From ac162401c56cd55a490dc9d387b27b9eb698c1c5 Mon Sep 17 00:00:00 2001 From: ruoshan Date: Mon, 2 Aug 2021 10:42:43 +0200 Subject: [PATCH 01/55] utilize eyetracking in sciview --- build.gradle.kts | 2 +- .../sc/iview/commands/file/OpenDirofTif.java | 79 ++ src/main/kotlin/sc/iview/SciView.kt | 18 + .../kotlin/sc/iview/commands/MenuWeights.kt | 1 + .../advanced/ConfirmableClickBehaviour.kt | 34 + .../commands/demo/advanced/EyeTrackingDemo.kt | 676 ++++++++++++++++++ .../demo/advanced/HedgehogAnalysis.kt | 369 ++++++++++ .../commands/demo/advanced/SpineMetadata.kt | 26 + 8 files changed, 1204 insertions(+), 1 deletion(-) create mode 100644 src/main/java/sc/iview/commands/file/OpenDirofTif.java create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt diff --git a/build.gradle.kts b/build.gradle.kts index bf9ec23f..ebbee3fd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,7 +33,7 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "e1e0b7e" + val sceneryVersion = "33cfcaa" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/src/main/java/sc/iview/commands/file/OpenDirofTif.java b/src/main/java/sc/iview/commands/file/OpenDirofTif.java new file mode 100644 index 00000000..9c0c0bf6 --- /dev/null +++ b/src/main/java/sc/iview/commands/file/OpenDirofTif.java @@ -0,0 +1,79 @@ +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) +public class OpenDirofTif implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "directory") + private File file; + + @Override + public void run() { + try { + sciView.openDirTiff(file.toPath()); + + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index cd8cb8c3..5deddf07 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -53,6 +53,7 @@ import graphics.scenery.utils.Statistics import graphics.scenery.utils.extensions.times import graphics.scenery.volumes.Colormap import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume import graphics.scenery.volumes.Volume.Companion.fromXML import graphics.scenery.volumes.Volume.Companion.setupId @@ -110,6 +111,7 @@ import java.io.File import java.io.IOException import java.nio.ByteBuffer import java.nio.FloatBuffer +import java.nio.file.Path import java.util.* import java.util.concurrent.Future import java.util.function.Consumer @@ -704,6 +706,22 @@ class SciView : SceneryBase, CalibratedRealInterval { } } + @Throws(IOException::class) + fun openDirTiff(source: Path) + { + val v = Volume.fromPath(source, hub) + v.name = "volume" + v.position = Vector3f(0.0f, 1.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.scale = Vector3f(10.0f, 10.0f,30.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = false + addNode(v) + } + + /** * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud * @param source string of a data source diff --git a/src/main/kotlin/sc/iview/commands/MenuWeights.kt b/src/main/kotlin/sc/iview/commands/MenuWeights.kt index 9abb35be..64b88ca1 100644 --- a/src/main/kotlin/sc/iview/commands/MenuWeights.kt +++ b/src/main/kotlin/sc/iview/commands/MenuWeights.kt @@ -111,6 +111,7 @@ object MenuWeights { const val DEMO_ADVANCED_CREMI = 1.0 const val DEMO_ADVANCED_BDVSLICING = 2.0 const val DEMO_ADVANCED_MESHTEXTURE = 3.0 + const val DEMO_ADVANCED_EYETRACKING =4.0 // Help const val HELP_HELP = 0.0 const val HELP_ABOUT = 200.0 diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt new file mode 100644 index 00000000..16ce340a --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt @@ -0,0 +1,34 @@ +package graphics.scenery.bionictracking + +import org.scijava.ui.behaviour.ClickBehaviour +import kotlin.concurrent.thread + + +/** + * [ClickBehaviour] that waits [timeout] for confirmation by re-executing the behaviour. + * Executes [armedAction] on first invocation, and [confirmAction] on second invocation, if + * it happens within [timeout]. + * + * @author Ulrik Guenther + */ +class ConfirmableClickBehaviour(val armedAction: (Long) -> Any, val confirmAction: (Long) -> Any, var timeout: Long = 3000): ClickBehaviour { + /** Whether the action is armed at the moment. Action becomes disarmed after [timeout]. */ + private var armed: Boolean = false + + /** + * Action fired at position [x]/[y]. Parameters not used in VR actions. + */ + override fun click(x : Int, y : Int) { + if(!armed) { + armed = true + armedAction.invoke(timeout) + + thread { + Thread.sleep(timeout) + armed = false + } + } else { + confirmAction.invoke(timeout) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt new file mode 100644 index 00000000..057bf6da --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -0,0 +1,676 @@ +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.numerics.OpenSimplexNoise +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize Eye Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class EyeTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + val hedgehogs = Mesh() + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + + referenceTarget.visible = false + referenceTarget.material.roughness = 1.0f + referenceTarget.material.metallic = 0.0f + referenceTarget.material.diffuse = Vector3f(0.8f, 0.8f, 0.8f) + sciview.camera!!.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material.roughness = 1.0f + calibrationTarget.material.metallic = 0.0f + calibrationTarget.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) + calibrationTarget.runRecursive { it.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) } + sciview.camera!!.addChild(calibrationTarget) + + laser.visible = false + laser.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.material.cullingMode = Material.CullingMode.Front + shell.material.diffuse = Vector3f(0.4f, 0.4f, 0.4f) + shell.position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + + volume = sciview.find("volume") as Volume + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.position = Vector3f(-1.0f, 1.5f, 0.0f) + left.rotation = left.rotation.rotationZ(PI.toFloat()) + //left.rotation = left.rotation.rotateByAngleZ(PI.toFloat()) + right.position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addChild(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.material.textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.allocateByteAndPut(data) + ) + + lastFrame = System.nanoTime() + } + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog -> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + hedgehog.instances.forEach { it.visible = true } + } + } + } + + + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.material.diffuse = Vector3f(0.5f, 0.5f, 0.5f) + sciview.camera!!.showMessage("Tracking deactivated.") + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog(parent: Node) { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + hedgehog.instancedProperties["ModelMatrix"] = { hedgehog.world } + hedgehog.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + parent.addChild(hedgehog) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden") + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint") + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible") + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s") + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s") + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing") + } else { + cam.showMessage("Paused") + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.", + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogs.children.removeAt(hedgehogs.children.size-1) + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.", + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}") + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", duration = 1000) + dumpHedgehog() + addHedgehog(hedgehogs) + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupCalibration() + + } + + + private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { + val startCalibration = ClickBehaviour { _, _ -> + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated) { + pupilTracker.onCalibrationInProgress = { + cam.showMessage("Crunching equations ...", messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage("Calibration failed.", messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage("Calibration succeeded!", messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) +// cam.children.find { it.name == "debugBoard" }?.visible = true + + for (i in 0 until 20) { + referenceTarget.material.diffuse = Vector3f(0.0f, 1.0f, 0.0f) + Thread.sleep(100) + referenceTarget.material.diffuse = Vector3f(0.8f, 0.8f, 0.8f) + Thread.sleep(30) + } + + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.material.diffuse = Vector3f(0.5f, 0.5f, 0.5f) + cam.showMessage("Tracking deactivated.") + dumpHedgehog() + } else { + addHedgehog(hedgehogs) + referenceTarget.material.diffuse = Vector3f(1.0f, 0.0f, 0.0f) + cam.showMessage("Tracking active.") + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + log.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", duration = 1500) + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + //NEW + PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { + log.info("Starting spine from $headCenter to $pointWorld") + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) + } + } + } + + else -> {gaze-> } + } + + log.info("Calibration routine done.") + } + + // bind calibration start to menu key on controller + + } + } + hmd.addBehaviour("start_calibration", startCalibration) + hmd.addKeyBinding("start_calibration", keybindingCalibration) + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) +// val p2 = center + direction * (sphereDist + 2.0f * sphere.radius) +// System.out.println(p1); +// System.out.println(p2); + val spine = Cylinder.betweenPoints(p1, p2, 1.0f, segments = 1) + spine.visible = false + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) +// System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry .add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit .add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + System.out.println(localEntry); + System.out.println(localExit); + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.world } + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + + hedgehogs.children.last().instances.add(spine) + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog(hedgehog: Node? = null) { + val lastHedgehog = hedgehog ?: hedgehogs.children.last() + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + log.info("Written hedgehog to ${hedgehogFile.absolutePath}") + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) + h.run() + } + + if(track == null) { +// log.warn("No track returned") + sciview.camera?.showMessage("No track returned", messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + + log.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = if(hedgehog == null) { + val m = Cylinder(0.005f, 1.0f, 10) + m.material = ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + m.material.diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + m.material.roughness = 1.0f + m.material.metallic = 0.0f + m.material.cullingMode = Material.CullingMode.None + m.instancedProperties["ModelMatrix"] = { m.world } + m.name = "Track-$hedgehogId" + m + } else { + null + } + + val parentId = 0 + val volumeDimension = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + if(master != null) { + val element = Mesh() + element.orientBetweenPoints(pair[0].first, pair[1].first, rescale = true, reposition = true) + element.parent = volume + element.instancedProperties["ModelMatrix"] = { element.world } + master.instances.add(element) + } + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimension))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + master?.let { volume.addChild(it) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt new file mode 100644 index 00000000..b043a197 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -0,0 +1,369 @@ +package graphics.scenery.bionictracking + +import org.joml.Vector3f +import org.joml.Vector4f +import org.joml.Matrix4f +//import com.jogamp.opengl.math.Quaternion +import org.joml.Quaternionf +import graphics.scenery.utils.LazyLogger +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import org.slf4j.LoggerFactory +import java.io.File +import kotlin.math.abs +import kotlin.math.pow +import kotlin.math.sqrt + +/** + * + * + * @author Ulrik Günther + */ +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { + val logger by LazyLogger() + + val timepoints = LinkedHashMap>() + + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set + + data class Track( + val points: List>, + val confidence: Float + ) + + init { + logger.info("Starting analysis with ${spines.size} spines") + + spines.forEach { spine -> + val timepoint = spine.timepoint + val current = timepoints[timepoint] + + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + private fun localMaxima(list: List): List> = + list.windowed(3, 1).mapIndexed { index, l -> + val left = l[0] + val center = l[1] + val right = l[2] + + // we have a match at center + if(left - center < 0 && center - right > 0) { + index + 1 to center + } else { + null + } + }.filterNotNull() + + data class SpineGraphVertex(val timepoint: Int, + val position: Vector3f, + val worldPosition: Vector3f, + val value: Float, + val metadata : SpineMetadata, + var previous: SpineGraphVertex? = null, + var next: SpineGraphVertex? = null) { + + fun distance(): Float { + val n = next + return if(n != null) { + val t = (n.worldPosition - this.worldPosition) + sqrt(t.x*t.x+t.y*t.y+t.z*t.z) + } else { + 0.0f + } + } + + fun drop() { + previous?.next = next + next?.previous = previous + } + + override fun toString() : String { + return "SpineGraphVertex for t=$timepoint, pos=$position, worldPos=$worldPosition, value=$value ($metadata)" + } + } + + fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) + + fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { + val cross = forward.cross(this) + val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) + + val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) + + return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) + } + + fun run(): Track? { + val startingThreshold = 0.02f + val localMaxThreshold = 0.01f + + if(timepoints.isEmpty()) { + return null + } + + val startingPoint = timepoints.entries.firstOrNull { entry -> + entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } + } ?: return null + + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + +// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) + + timepoints.filter { it.key < startingPoint.key } + .forEach { timepoints.remove(it.key) } + + logger.info("${timepoints.size} timepoints left") + + val candidates = timepoints.map { tp -> + val vs = tp.value.mapIndexedNotNull { i, spine -> + val maxIndices = localMaxima(spine.samples.filterNotNull()) + logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + + // compare gaze and head orientation + +// if(spine.headOrientation.dot(spine.direction.toQuaternion()) > 0.0f) { +// return@mapIndexedNotNull null +// } + + if(maxIndices.isNotEmpty()) { + maxIndices.map { index -> + val position = spine.localEntry.add(spine.localDirection).mul(index.first.toFloat()) + val worldPosition = localToWorld.transform((position.mul(2.0f) - Vector3f(1.0f)).xyzw()).xyz() +// val worldPosition = localToWorld.mult((position * 2.0f - GLVector.getOneVector(3)).xyzw()).xyz() + + SpineGraphVertex(tp.key, + position, + worldPosition, + index.second, + spine) + } + } else { + null + } + } + vs + }.flatten() + + + // get the initial vertex, this one is assumed to always be in front, and have a local max + val initial = candidates.first().first() + var current = initial + + var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> + val distances = vs + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = sqrt(t.x*t.x+t.y*t.y+t.z*t.z) + vertex to distance + } + .sortedBy { it.second } + + logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second}") + + // select closest vertex + val closest = distances.firstOrNull()?.first + if(closest != null) { + current.next = closest + closest.previous = current + current = closest + current + } else { + null + } + }.toMutableList() + + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = (value - m)/sd + + val beforeCount = shortestPath.size + if(false) { + while (shortestPath.any { it.distance() >= 2.0f * avgPathLength }) { + shortestPath.filter { it.distance() >= 2.0f * avgPathLength }.forEach { it.drop() } + shortestPath = shortestPath.filter { it.distance() < 2.0f * avgPathLength }.toMutableList() + } + } else { + // logger.info("Distances: ${shortestPath.joinToString { "${it.distance()}/${zScore(it.distance(), avgPathLength, stdDevPathLength)}"}}") + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } + while(remaining > 0) { + val outliers = shortestPath + .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } + .map { + val idx = shortestPath.indexOf(it) +// listOf(idx, idx+1) + listOf(idx-1, idx, idx+1) + }.flatten() + + shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() + remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } +// indices.forEach { +// shortestPath.removeAt(it.second) +// } + +// shortestPath = shortestPath.filter { zScore(it.distance(), avgPathLength, stdDevPathLength) <= 2.0f } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + +// avgPathLength = shortestPath.map { it.distance() }.average().toFloat() +// stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + } + } + val afterCount = shortestPath.size + logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") + logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") + + val singlePoints = shortestPath + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull { it.metadata.confidence } } + .filter { + it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f + } + + + logger.info("Returning ${singlePoints.size} points") + + return Track(singlePoints.map { it.position . mul (2.0f) - Vector3f(1.0f) to it }, avgConfidence) + } + + companion object { + private val logger by LazyLogger() + + fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { + logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val confidence = tokens[1].toFloat() + val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + Vector3f(0.0f), + Vector3f(0.0f), + 0.0f, + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Quaternionf(), + Vector3f(0.0f), + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + + private fun String.toVector3f(): Vector3f { + val array = this + .trim() + .trimEnd() + .replace("[[", "").replace("]]", "") + .split(",") + .map { it.trim().trimEnd().toFloat() }.toFloatArray() + + return Vector3f(array[0],array[1],array[2]) + } + + private fun String.toQuaternionf(): Quaternionf { + val array = this + .trim() + .trimEnd() + .replace("Quaternion[", "").replace("]", "") + .split(",") + .map { it.trim().trimEnd().replace("x ", "").replace("y ", "").replace("z ","").replace("w ", "").toFloat() } + + return Quaternionf(array[0], array[1], array[2], array[3]) + } + + fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + } +} + +fun main(args: Array) { + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") + if(args.isEmpty()) { + logger.error("Sorry, but a file name is needed.") + return + } + + val file = File(args[0]) + val analysis = HedgehogAnalysis.fromIncompleteCSV(file) + + val results = analysis.run() + logger.info("Results: \n$results") +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt new file mode 100644 index 00000000..d1ebca32 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt @@ -0,0 +1,26 @@ +package graphics.scenery.bionictracking + +//import cleargl.GLMatrix +//import cleargl.GLVector +import com.jogamp.opengl.math.Quaternion +import org.joml.Quaternionf +import org.joml.Vector3f + +/** + * Data class to store metadata for spines of the hedgehog. + */ +data class SpineMetadata( + val timepoint: Int, + val origin: Vector3f, + val direction: Vector3f, + val distance: Float, + val localEntry: Vector3f, + val localExit: Vector3f, + val localDirection: Vector3f, + val headPosition: Vector3f, + val headOrientation: Quaternionf, +// val headOrientation: Quaternion, + val position: Vector3f, + val confidence: Float, + val samples: List +) \ No newline at end of file From c3bc1b4a3263df36438422ecd4ebc58233fc0904 Mon Sep 17 00:00:00 2001 From: ruoshan Date: Thu, 9 Sep 2021 17:38:23 +0200 Subject: [PATCH 02/55] a working version and adjust the size of some messages --- build.gradle.kts | 2 +- src/main/kotlin/sc/iview/SciView.kt | 6 - .../commands/demo/advanced/EyeTrackingDemo.kt | 58 ++-- .../demo/advanced/HedgehogAnalysis.kt | 326 ++++++++++-------- 4 files changed, 214 insertions(+), 178 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ebbee3fd..6fe3604e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,7 +33,7 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "33cfcaa" + val sceneryVersion = "6076983" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 5deddf07..163aaf97 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -711,12 +711,6 @@ class SciView : SceneryBase, CalibratedRealInterval { { val v = Volume.fromPath(source, hub) v.name = "volume" - v.position = Vector3f(0.0f, 1.0f, 0.0f) - v.colormap = Colormap.get("jet") - v.scale = Vector3f(10.0f, 10.0f,30.0f) - v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) - v.metadata["animating"] = true - v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) v.visible = false addNode(v) } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 057bf6da..8cb095ab 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -72,7 +72,7 @@ class EyeTrackingDemo: Command{ @Parameter private lateinit var log: LogService - + val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) lateinit var hmd: OpenVRHMD val referenceTarget = Icosphere(0.004f, 2) @@ -137,6 +137,13 @@ class EyeTrackingDemo: Command{ volume = sciview.find("volume") as Volume + volume.position = Vector3f(0.0f, 1.0f, 0.0f) + volume.colormap = Colormap.get("jet") + volume.scale = Vector3f(10.0f, 10.0f,30.0f) + volume.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + volume.metadata["animating"] = true + volume.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + volume.visible = false sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -152,7 +159,6 @@ class EyeTrackingDemo: Command{ val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) left.position = Vector3f(-1.0f, 1.5f, 0.0f) left.rotation = left.rotation.rotationZ(PI.toFloat()) - //left.rotation = left.rotation.rotateByAngleZ(PI.toFloat()) right.position = Vector3f(1.0f, 1.5f, 0.0f) eyeFrames.addChild(left) eyeFrames.addChild(right) @@ -209,9 +215,9 @@ class EyeTrackingDemo: Command{ } } } - thread{ - inputSetup() - } + thread{ + inputSetup() + } thread { while(!sciview.isInitialized) { Thread.sleep(200) } @@ -251,8 +257,6 @@ class EyeTrackingDemo: Command{ } } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { tracking = false @@ -350,9 +354,9 @@ class EyeTrackingDemo: Command{ val playPause = ClickBehaviour { _, _ -> playing = !playing if(playing) { - cam.showMessage("Playing") + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) } else { - cam.showMessage("Paused") + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) } } @@ -480,7 +484,7 @@ class EyeTrackingDemo: Command{ sciview.deleteNode(sciview.find("eyeFrames")) log.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", duration = 1500) + cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) pupilTracker.calibrate(cam, hmd, generateReferenceData = true, calibrationTarget = calibrationTarget) @@ -500,13 +504,13 @@ class EyeTrackingDemo: Command{ val direction = (pointWorld - headCenter).normalize() if (tracking) { - log.info("Starting spine from $headCenter to $pointWorld") +// log.info("Starting spine from $headCenter to $pointWorld") addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) } } } - else -> {gaze-> } +// else -> {gaze-> } } log.info("Calibration routine done.") @@ -529,20 +533,16 @@ class EyeTrackingDemo: Command{ val p1 = center val temp = direction.mul(sphereDist + 2.0f * sphere.radius) val p2 = Vector3f(center).add(temp) -// val p2 = center + direction * (sphereDist + 2.0f * sphere.radius) -// System.out.println(p1); -// System.out.println(p2); + val spine = Cylinder.betweenPoints(p1, p2, 1.0f, segments = 1) - spine.visible = false + spine.visible = true val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) // System.out.println(intersection); if(intersection is MaybeIntersects.Intersection) { // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry .add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit .add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - System.out.println(localEntry); - System.out.println(localExit); + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null if (samples != null && localDirection != null) { @@ -598,22 +598,20 @@ class EyeTrackingDemo: Command{ } spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") } hedgehogFileWriter.close() - log.info("Written hedgehog to ${hedgehogFile.absolutePath}") - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track val track = if(existingAnalysis is HedgehogAnalysis.Track) { existingAnalysis } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) h.run() } if(track == null) { -// log.warn("No track returned") +// logger.warn("No track returned") sciview.camera?.showMessage("No track returned", messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) return } @@ -621,10 +619,10 @@ class EyeTrackingDemo: Command{ lastHedgehog.metadata["HedgehogAnalysis"] = track lastHedgehog.metadata["Spines"] = spines - log.info("---\nTrack: ${track.points.joinToString("\n")}\n---") +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") val master = if(hedgehog == null) { - val m = Cylinder(0.005f, 1.0f, 10) + val m = Cylinder(3f, 1.0f, 10) m.material = ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") m.material.diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) m.material.roughness = 1.0f @@ -638,7 +636,7 @@ class EyeTrackingDemo: Command{ } val parentId = 0 - val volumeDimension = volume.getDimensions() + val volumeDimensions = volume.getDimensions() trackFileWriter.newLine() trackFileWriter.newLine() @@ -646,13 +644,13 @@ class EyeTrackingDemo: Command{ track.points.windowed(2, 1).forEach { pair -> if(master != null) { val element = Mesh() - element.orientBetweenPoints(pair[0].first, pair[1].first, rescale = true, reposition = true) + element.orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) element.parent = volume element.instancedProperties["ModelMatrix"] = { element.world } master.instances.add(element) } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimension))//direct product + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index b043a197..3c03273a 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -1,16 +1,21 @@ package graphics.scenery.bionictracking +import graphics.scenery.Icosphere +import graphics.scenery.Scene +import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f import org.joml.Vector3f import org.joml.Vector4f import org.joml.Matrix4f //import com.jogamp.opengl.math.Quaternion import org.joml.Quaternionf import graphics.scenery.utils.LazyLogger -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.utils.extensions.* +import jdk.internal.cmm.SystemResourcePressureImpl import org.slf4j.LoggerFactory import java.io.File +import java.util.* +import kotlin.collections.ArrayList +import kotlin.collections.LinkedHashMap import kotlin.math.abs import kotlin.math.pow import kotlin.math.sqrt @@ -20,58 +25,59 @@ import kotlin.math.sqrt * * @author Ulrik Günther */ -class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { - val logger by LazyLogger() +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f, val dimension : Vector3f) { + val logger by LazyLogger() - val timepoints = LinkedHashMap>() + val timepoints = LinkedHashMap>() - var avgConfidence = 0.0f - private set - var totalSampleCount = 0 - private set + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set - data class Track( - val points: List>, - val confidence: Float - ) + data class Track( + val points: List>, + val confidence: Float + ) - init { + init { logger.info("Starting analysis with ${spines.size} spines") spines.forEach { spine -> val timepoint = spine.timepoint val current = timepoints[timepoint] - if(current == null) { - timepoints[timepoint] = arrayListOf(spine) - } else { - current.add(spine) - } - - avgConfidence += spine.confidence - totalSampleCount++ - } - - avgConfidence /= totalSampleCount - } - - private fun localMaxima(list: List): List> = - list.windowed(3, 1).mapIndexed { index, l -> - val left = l[0] - val center = l[1] - val right = l[2] - - // we have a match at center - if(left - center < 0 && center - right > 0) { - index + 1 to center - } else { - null - } - }.filterNotNull() + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + private fun localMaxima(list: List): List> = + list.windowed(3, 1).mapIndexed { index, l -> + val left = l[0] + val center = l[1] + val right = l[2] + + // we have a match at center + if(left - center < 0 && center - right > 0) { + index + 1 to center + } else { + null + } + }.filterNotNull() data class SpineGraphVertex(val timepoint: Int, val position: Vector3f, val worldPosition: Vector3f, + val index: Int, val value: Float, val metadata : SpineMetadata, var previous: SpineGraphVertex? = null, @@ -81,7 +87,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val n = next return if(n != null) { val t = (n.worldPosition - this.worldPosition) - sqrt(t.x*t.x+t.y*t.y+t.z*t.z) + sqrt(t.x*t.x + t.y*t.y + t.z*t.z) } else { 0.0f } @@ -93,11 +99,12 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } override fun toString() : String { - return "SpineGraphVertex for t=$timepoint, pos=$position, worldPos=$worldPosition, value=$value ($metadata)" + return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" } } fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) +// fun Iterable.avg() = (this.map { it}.sum() / this.count()) fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { val cross = forward.cross(this) @@ -108,15 +115,16 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) } - fun run(): Track? { - val startingThreshold = 0.02f + fun run(): Track? { + val startingThreshold = 0.02f val localMaxThreshold = 0.01f + val zscoreThreshold = 2.0f if(timepoints.isEmpty()) { return null } - val startingPoint = timepoints.entries.firstOrNull { entry -> + val startingPoint = timepoints.entries.firstOrNull { entry -> entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } } ?: return null @@ -124,59 +132,58 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) - timepoints.filter { it.key < startingPoint.key } + timepoints.filter { it.key > startingPoint.key } .forEach { timepoints.remove(it.key) } logger.info("${timepoints.size} timepoints left") val candidates = timepoints.map { tp -> - val vs = tp.value.mapIndexedNotNull { i, spine -> - val maxIndices = localMaxima(spine.samples.filterNotNull()) - logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") - - // compare gaze and head orientation - -// if(spine.headOrientation.dot(spine.direction.toQuaternion()) > 0.0f) { -// return@mapIndexedNotNull null -// } + val vs = tp.value.mapIndexedNotNull { i, spine -> + val maxIndices = localMaxima(spine.samples.filterNotNull()) + logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") if(maxIndices.isNotEmpty()) { - maxIndices.map { index -> - val position = spine.localEntry.add(spine.localDirection).mul(index.first.toFloat()) - val worldPosition = localToWorld.transform((position.mul(2.0f) - Vector3f(1.0f)).xyzw()).xyz() -// val worldPosition = localToWorld.mult((position * 2.0f - GLVector.getOneVector(3)).xyzw()).xyz() + maxIndices.filter { it.first <1200}. + map { index -> +// logger.info(index.toString()) + val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) + val worldPosition = localToWorld.transform((Vector3f(position).mul(dimension)).xyzw()).xyz() SpineGraphVertex(tp.key, position, worldPosition, + index.first, index.second, spine) + } } else { null } - } + } vs - }.flatten() + }.flatten() + // get the initial vertex, this one is assumed to always be in front, and have a local max - val initial = candidates.first().first() + val initial = candidates.first().filter{it.value>startingThreshold}.first() + System.out.println("initial:"+initial) var current = initial - var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> +// System.out.println(time) + val distances = vs .filter { it.value > localMaxThreshold } .map { vertex -> val t = current.worldPosition - vertex.worldPosition - val distance = sqrt(t.x*t.x+t.y*t.y+t.z*t.z) + val distance = t.length() vertex to distance } .sortedBy { it.second } +// logger.info("distances - ${distances}") +// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") - logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second}") - - // select closest vertex val closest = distances.firstOrNull()?.first if(closest != null) { current.next = closest @@ -188,63 +195,67 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } }.toMutableList() + + + var check = shortestPath + + + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) + val beforeCount = shortestPath.size + System.out.println("before short path:"+ shortestPath.size) + + while (shortestPath.any { it.distance() >= 2.0f * avgPathLength }) { + shortestPath.filter { it.distance() >= 2.0f * avgPathLength }.forEach { it.drop() } + shortestPath = shortestPath.filter { it.distance() < 2.0f * avgPathLength }.toMutableList() + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + } + shortestPath.windowed(3, 1, partialWindows = true).forEach { it.getOrNull(0)?.next = it.getOrNull(1) it.getOrNull(1)?.previous = it.getOrNull(0) it.getOrNull(1)?.next = it.getOrNull(2) it.getOrNull(2)?.previous = it.getOrNull(1) } + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } +// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + while(remaining > 0) { + val outliers = shortestPath + .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + .map { + val idx = shortestPath.indexOf(it) + listOf(idx-1,idx,idx+1) + }.flatten() - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - fun zScore(value: Float, m: Float, sd: Float) = (value - m)/sd +// logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - val beforeCount = shortestPath.size - if(false) { - while (shortestPath.any { it.distance() >= 2.0f * avgPathLength }) { - shortestPath.filter { it.distance() >= 2.0f * avgPathLength }.forEach { it.drop() } - shortestPath = shortestPath.filter { it.distance() < 2.0f * avgPathLength }.toMutableList() - } - } else { - // logger.info("Distances: ${shortestPath.joinToString { "${it.distance()}/${zScore(it.distance(), avgPathLength, stdDevPathLength)}"}}") - var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } - while(remaining > 0) { - val outliers = shortestPath - .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } - .map { - val idx = shortestPath.indexOf(it) -// listOf(idx, idx+1) - listOf(idx-1, idx, idx+1) - }.flatten() - - shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > 2.0f } -// indices.forEach { -// shortestPath.removeAt(it.second) -// } - -// shortestPath = shortestPath.filter { zScore(it.distance(), avgPathLength, stdDevPathLength) <= 2.0f } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } -// avgPathLength = shortestPath.map { it.distance() }.average().toFloat() -// stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) } +// avgPathLength = shortestPath.map { it.distance() }.average().toFloat() +// stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") } + val afterCount = shortestPath.size logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") - logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") - +// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") +// logger.info(shortestPath.toString()) val singlePoints = shortestPath .groupBy { it.timepoint } - .mapNotNull { vs -> vs.value.maxByOrNull { it.metadata.confidence } } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } .filter { it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f } @@ -252,8 +263,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix logger.info("Returning ${singlePoints.size} points") - return Track(singlePoints.map { it.position . mul (2.0f) - Vector3f(1.0f) to it }, avgConfidence) - } + return Track(singlePoints.map { it.position to it}, avgConfidence) + } companion object { private val logger by LazyLogger() @@ -287,29 +298,62 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix spines.add(currentSpine) } - return HedgehogAnalysis(spines, Matrix4f()) + return HedgehogAnalysis(spines, Matrix4f(), Vector3f()) } private fun String.toVector3f(): Vector3f { - val array = this - .trim() - .trimEnd() - .replace("[[", "").replace("]]", "") - .split(",") - .map { it.trim().trimEnd().toFloat() }.toFloatArray() - - return Vector3f(array[0],array[1],array[2]) +// System.out.println(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + + if (array[0] == "+Inf" || array[0] == "-Inf") + return Vector3f(0.0f,0.0f,0.0f) + + return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) } private fun String.toQuaternionf(): Quaternionf { - val array = this - .trim() - .trimEnd() - .replace("Quaternion[", "").replace("]", "") - .split(",") - .map { it.trim().trimEnd().replace("x ", "").replace("y ", "").replace("z ","").replace("w ", "").toFloat() } - - return Quaternionf(array[0], array[1], array[2], array[3]) +// System.out.println(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) + } + fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f, dimension: Vector3f, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, matrix4f,dimension) } fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { @@ -349,21 +393,21 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix spines.add(currentSpine) } - return HedgehogAnalysis(spines, Matrix4f()) + return HedgehogAnalysis(spines, Matrix4f(),Vector3f()) } } } fun main(args: Array) { - val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") - if(args.isEmpty()) { - logger.error("Sorry, but a file name is needed.") - return - } - - val file = File(args[0]) - val analysis = HedgehogAnalysis.fromIncompleteCSV(file) - - val results = analysis.run() - logger.info("Results: \n$results") -} \ No newline at end of file + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") +// if(args.isEmpty()) { +// logger.error("Sorry, but a file name is needed.") +// return +// } + + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone-2021-08-10 16.17.40\\Hedgehog_1_2021-08-10 16.21.35.csv") +// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") +} From f1f00c7b17a3b7590e90b0d76efbb31517c7910a Mon Sep 17 00:00:00 2001 From: ruoshan Date: Thu, 9 Sep 2021 18:25:05 +0200 Subject: [PATCH 03/55] code clean --- .../sc/iview/commands/demo/advanced/HedgehogAnalysis.kt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 3c03273a..7bfb4d15 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -6,16 +6,11 @@ import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f import org.joml.Vector3f import org.joml.Vector4f import org.joml.Matrix4f -//import com.jogamp.opengl.math.Quaternion import org.joml.Quaternionf import graphics.scenery.utils.LazyLogger import graphics.scenery.utils.extensions.* -import jdk.internal.cmm.SystemResourcePressureImpl import org.slf4j.LoggerFactory import java.io.File -import java.util.* -import kotlin.collections.ArrayList -import kotlin.collections.LinkedHashMap import kotlin.math.abs import kotlin.math.pow import kotlin.math.sqrt @@ -26,7 +21,7 @@ import kotlin.math.sqrt * @author Ulrik Günther */ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f, val dimension : Vector3f) { - val logger by LazyLogger() +// val logger by LazyLogger() val timepoints = LinkedHashMap>() From 9eb458e565a6ba243e8837aa1d9905d8984d567a Mon Sep 17 00:00:00 2001 From: ruoshan Date: Thu, 7 Oct 2021 10:22:32 +0200 Subject: [PATCH 04/55] let the volume show before starting the eye tracking --- src/main/kotlin/sc/iview/SciView.kt | 39 +++++++++++-------- .../commands/demo/advanced/EyeTrackingDemo.kt | 19 ++------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 163aaf97..4c6b8b94 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -711,8 +711,14 @@ class SciView : SceneryBase, CalibratedRealInterval { { val v = Volume.fromPath(source, hub) v.name = "volume" - v.visible = false - addNode(v) + v.position = Vector3f(0.0f, 1.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.scale = Vector3f(10.0f, 10.0f,30.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = true + addChild(v) } @@ -1512,20 +1518,21 @@ class SciView : SceneryBase, CalibratedRealInterval { } renderer!!.pushMode = false // we need to force reloading the renderer as the HMD might require device or instance extensions - if (renderer is VulkanRenderer && hmdAdded) { - replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) - (renderer as VulkanRenderer).toggleVR() - while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { - logger.debug("Waiting for renderer reinitialisation") - try { - Thread.sleep(200) - } catch (e: InterruptedException) { - e.printStackTrace() - } - } - } else { - renderer!!.toggleVR() - } +// if (renderer is VulkanRenderer && hmdAdded) { +// replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) +// (renderer as VulkanRenderer).toggleVR() +// while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { +// logger.debug("Waiting for renderer reinitialisation") +// try { +// Thread.sleep(200) +// } catch (e: InterruptedException) { +// e.printStackTrace() +// } +// } +// } else { +// renderer!!.toggleVR() +// } + renderer!!.toggleVR() } /** diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 8cb095ab..41a46e0d 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -108,6 +108,10 @@ class EyeTrackingDemo: Command{ override fun run() { + volume = sciview.find("volume") as Volume + volume.visible = false + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) @@ -135,19 +139,6 @@ class EyeTrackingDemo: Command{ shell.position = Vector3f(0.0f, 0.0f, 0.0f) sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - volume.position = Vector3f(0.0f, 1.0f, 0.0f) - volume.colormap = Colormap.get("jet") - volume.scale = Vector3f(10.0f, 10.0f,30.0f) - volume.transferFunction = TransferFunction.ramp(0.05f, 0.8f) - volume.metadata["animating"] = true - volume.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) - volume.visible = false - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - val bb = BoundingGrid() bb.node = volume bb.visible = false @@ -168,7 +159,6 @@ class EyeTrackingDemo: Command{ val pupilFrameLimit = 20 var lastFrame = System.nanoTime() - pupilTracker.subscribeFrames { eye, texture -> if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { return@subscribeFrames @@ -218,7 +208,6 @@ class EyeTrackingDemo: Command{ thread{ inputSetup() } - thread { while(!sciview.isInitialized) { Thread.sleep(200) } From 785eb2c3f6f9288c5a74a5e15248b406b66009a6 Mon Sep 17 00:00:00 2001 From: ruoshan Date: Tue, 12 Oct 2021 13:17:24 +0200 Subject: [PATCH 05/55] update bionic tracking --- build.gradle.kts | 2 +- .../commands/demo/advanced/EyeTrackingDemo.kt | 100 +++++++++++------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 6fe3604e..f686c24e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,7 +33,7 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "6076983" + val sceneryVersion = "19edc3a221cb1684b6472624d5c70f340b8645a3" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 41a46e0d..dd332126 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -60,6 +60,9 @@ import net.imglib2.img.Img import net.imglib2.view.Views import org.lwjgl.openvr.OpenVR import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard @Plugin(type = Command::class, menuRoot = "SciView", @@ -84,6 +87,7 @@ class EyeTrackingDemo: Command{ lateinit var sessionDirectory: Path val hedgehogs = Mesh() + val hedgehogsInstanced = InstancedNode(hedgehogs) enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } var hedgehogVisibility = HedgehogVisibility.Hidden @@ -115,28 +119,31 @@ class EyeTrackingDemo: Command{ sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - referenceTarget.visible = false - referenceTarget.material.roughness = 1.0f - referenceTarget.material.metallic = 0.0f - referenceTarget.material.diffuse = Vector3f(0.8f, 0.8f, 0.8f) + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } sciview.camera!!.addChild(referenceTarget) calibrationTarget.visible = false - calibrationTarget.material.roughness = 1.0f - calibrationTarget.material.metallic = 0.0f - calibrationTarget.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) - calibrationTarget.runRecursive { it.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) } + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f)} sciview.camera!!.addChild(calibrationTarget) laser.visible = false - laser.material.diffuse = Vector3f(1.0f, 1.0f, 1.0f) + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} sciview.addChild(laser) val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.material.cullingMode = Material.CullingMode.Front - shell.material.diffuse = Vector3f(0.4f, 0.4f, 0.4f) - shell.position = Vector3f(0.0f, 0.0f, 0.0f) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) sciview.addChild(shell) val bb = BoundingGrid() @@ -148,9 +155,9 @@ class EyeTrackingDemo: Command{ val eyeFrames = Mesh("eyeFrames") val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) - left.position = Vector3f(-1.0f, 1.5f, 0.0f) - left.rotation = left.rotation.rotationZ(PI.toFloat()) - right.position = Vector3f(1.0f, 1.5f, 0.0f) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) eyeFrames.addChild(left) eyeFrames.addChild(right) @@ -174,12 +181,12 @@ class EyeTrackingDemo: Command{ val image = ImageIO.read(stream) val data = (image.raster.dataBuffer as DataBufferByte).data - node.material.textures["diffuse"] = Texture( + node.ifMaterial {textures["diffuse"] = Texture( Vector3i(image.width, image.height, 1), 3, UnsignedByteType(), BufferUtils.allocateByteAndPut(data) - ) + ) } lastFrame = System.nanoTime() } @@ -235,13 +242,15 @@ class EyeTrackingDemo: Command{ if(hedgehogs.visible) { if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { hedgehogs.children.forEach { hedgehog -> - hedgehog.instances.forEach { + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instances.forEach { it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint } } } else { hedgehogs.children.forEach { hedgehog -> - hedgehog.instances.forEach { it.visible = true } + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instances.forEach { it.visible = true } } } } @@ -249,7 +258,7 @@ class EyeTrackingDemo: Command{ if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { tracking = false - referenceTarget.material.diffuse = Vector3f(0.5f, 0.5f, 0.5f) + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} sciview.camera!!.showMessage("Tracking deactivated.") dumpHedgehog() } @@ -266,9 +275,10 @@ class EyeTrackingDemo: Command{ hedgehog.visible = false // hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, // listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - hedgehog.instancedProperties["ModelMatrix"] = { hedgehog.world } - hedgehog.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - parent.addChild(hedgehog) + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + parent.addChild(hedgehogInstanced) } @@ -440,9 +450,9 @@ class EyeTrackingDemo: Command{ // cam.children.find { it.name == "debugBoard" }?.visible = true for (i in 0 until 20) { - referenceTarget.material.diffuse = Vector3f(0.0f, 1.0f, 0.0f) + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} Thread.sleep(100) - referenceTarget.material.diffuse = Vector3f(0.8f, 0.8f, 0.8f) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} Thread.sleep(30) } @@ -451,12 +461,12 @@ class EyeTrackingDemo: Command{ val toggleTracking = ClickBehaviour { _, _ -> if (tracking) { - referenceTarget.material.diffuse = Vector3f(0.5f, 0.5f, 0.5f) + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } cam.showMessage("Tracking deactivated.") dumpHedgehog() } else { addHedgehog(hedgehogs) - referenceTarget.material.diffuse = Vector3f(1.0f, 0.0f, 0.0f) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } cam.showMessage("Tracking active.") } tracking = !tracking @@ -552,10 +562,13 @@ class EyeTrackingDemo: Command{ val count = samples.filterNotNull().count { it > 0.2f } spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.world } - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + val spineInstanced = InstancedNode(spine) + spineInstanced.instancedProperties["ModelMatrix"] = { spine.spatial().world } + spineInstanced.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + + val hedgehogInstanced = InstancedNode(hedgehogs.children.last()) + hedgehogInstanced.instances.add(spineInstanced.addInstance()) - hedgehogs.children.last().instances.add(spine) } } } @@ -582,7 +595,8 @@ class EyeTrackingDemo: Command{ trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") } - val spines = lastHedgehog.instances.mapNotNull { spine -> + val lastHedgehogInstanced = InstancedNode(lastHedgehog) + val spines = lastHedgehogInstanced.instances.mapNotNull { spine -> spine.metadata["spine"] as? SpineMetadata } @@ -612,14 +626,17 @@ class EyeTrackingDemo: Command{ val master = if(hedgehog == null) { val m = Cylinder(3f, 1.0f, 10) - m.material = ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - m.material.diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - m.material.roughness = 1.0f - m.material.metallic = 0.0f - m.material.cullingMode = Material.CullingMode.None - m.instancedProperties["ModelMatrix"] = { m.world } + m.ifMaterial { + ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + roughness = 1.0f + metallic = 0.0f + cullingMode = Material.CullingMode.None + } m.name = "Track-$hedgehogId" - m + val mInstanced = InstancedNode(m) + mInstanced.instancedProperties["ModelMatrix"] = { m.spatial().world } + mInstanced } else { null } @@ -633,10 +650,11 @@ class EyeTrackingDemo: Command{ track.points.windowed(2, 1).forEach { pair -> if(master != null) { val element = Mesh() - element.orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) element.parent = volume - element.instancedProperties["ModelMatrix"] = { element.world } - master.instances.add(element) + val elementInstanced = InstancedNode(element) + elementInstanced.instancedProperties["ModelMatrix"] = { element.spatial().world } + master.instances.add(elementInstanced.addInstance()) } val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product From b4f71887f78efc1cb017d0a605367be72f16c32e Mon Sep 17 00:00:00 2001 From: ruoshan Date: Mon, 15 Nov 2021 11:38:09 +0100 Subject: [PATCH 06/55] add headset for cell tracking --- build.gradle.kts | 2 +- .../sc/iview/commands/file/OpenTrackFile.java | 79 +++ src/main/kotlin/sc/iview/SciView.kt | 158 ++++- .../demo/advanced/ControllerTrackingDemo.kt | 593 ++++++++++++++++++ .../commands/demo/advanced/EyeTrackingDemo.kt | 85 ++- 5 files changed, 853 insertions(+), 64 deletions(-) create mode 100644 src/main/java/sc/iview/commands/file/OpenTrackFile.java create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt diff --git a/build.gradle.kts b/build.gradle.kts index 332921ca..b3c5304d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,7 +36,7 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "19edc3a221cb1684b6472624d5c70f340b8645a3" + val sceneryVersion = "37fc0836c8" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/src/main/java/sc/iview/commands/file/OpenTrackFile.java b/src/main/java/sc/iview/commands/file/OpenTrackFile.java new file mode 100644 index 00000000..b2c3e7a7 --- /dev/null +++ b/src/main/java/sc/iview/commands/file/OpenTrackFile.java @@ -0,0 +1,79 @@ +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Track File", weight = FILE_OPEN) }) +public class OpenTrackFile implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "open,extensions:csv") + private File file; + + @Override + public void run() { + try { + sciView.openTrackFile(file); + + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 5863e079..84a177ce 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -45,7 +45,9 @@ import graphics.scenery.backends.opengl.OpenGLRenderer import graphics.scenery.backends.vulkan.VulkanRenderer import graphics.scenery.controls.InputHandler import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedStereoGlasses import graphics.scenery.controls.TrackerInput +import graphics.scenery.numerics.Random import graphics.scenery.primitives.* import graphics.scenery.proteins.Protein import graphics.scenery.proteins.RibbonDiagram @@ -106,6 +108,7 @@ import sc.iview.event.NodeActivatedEvent import sc.iview.event.NodeAddedEvent import sc.iview.event.NodeChangedEvent import sc.iview.event.NodeRemovedEvent +import sc.iview.node.Line3D import sc.iview.process.MeshConverter import sc.iview.ui.CustomPropertyUI import sc.iview.ui.MainWindow @@ -113,6 +116,7 @@ import sc.iview.ui.SwingMainWindow import sc.iview.ui.TaskManager import tpietzsch.example2.VolumeViewerOptions import java.awt.event.WindowListener +import java.io.File import java.io.IOException import java.nio.ByteBuffer import java.nio.FloatBuffer @@ -123,8 +127,12 @@ import java.util.function.Consumer import java.util.function.Function import java.util.function.Predicate import java.util.stream.Collectors +import kotlin.collections.ArrayList import kotlin.math.cos import kotlin.math.sin +import kotlin.system.measureTimeMillis +import org.joml.Vector4f +import sc.iview.commands.demo.animation.ParticleDemo /** * Main SciView class. @@ -714,9 +722,9 @@ class SciView : SceneryBase, CalibratedRealInterval { { val v = Volume.fromPath(source, hub) v.name = "volume" - v.position = Vector3f(0.0f, 1.0f, 0.0f) + v.spatial().position = Vector3f(0.0f, 1.0f, 0.0f) v.colormap = Colormap.get("jet") - v.scale = Vector3f(10.0f, 10.0f,30.0f) + v.spatial().scale = Vector3f(10.0f, 10.0f,30.0f) v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) v.metadata["animating"] = true v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) @@ -724,6 +732,122 @@ class SciView : SceneryBase, CalibratedRealInterval { addChild(v) } + data class PointInTrack( + val t: Int, + val loc: Vector3f, + val cellId: Long, + val parentId: Long, + val nodeScore: Float, + val edgeScore: Float + ) + + data class Track( + val track: List, + val trackId: Int + ) + + @Throws(IOException::class) + fun openTrackFile(file: File) + { + val lines = file.readLines() + var track = ArrayList() + val tracks = ArrayList() + val separator = "," + + var lastTrackId = -1 + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val t = tokens[0].toInt() + val z = tokens[1].toFloat() -2000f + val y = tokens[2].toFloat() -800f + val x = tokens[3].toFloat() -1300f + val cellId = tokens[4].toLong() + val parentId = tokens[5].toLong() + val trackId = tokens[6].toInt() + val nodeScore = tokens[7].toFloat() + val edgeScore = tokens[8].toFloat()/45.0f + + val currentPointInTrack = PointInTrack( + t, + Vector3f(x,y,z), + cellId, + parentId, + nodeScore, + edgeScore + ) +// System.out.println(currentPointInTrack) + if(lastTrackId != trackId) + { +// Thread.sleep(2000) + lastTrackId = trackId +// System.out.println("trackId: "+trackId.toString()) + val sortedTrack = track.sortedBy { it.t } + tracks.add(Track(sortedTrack, trackId)) +// System.out.println(sortedTrack) + + track.clear() + } + + track.add(currentPointInTrack) + + } + val timeCost = measureTimeMillis { + addTracks(tracks) + } + println("time: $timeCost") + } + + + + fun addTracks(tracks: ArrayList) + { + val rng = Random(17) + for(track in tracks) + { + if(track.trackId > 10) + { +// continue + } +// Thread.sleep(2000) + System.out.println("add track: "+ track.trackId.toString() ) +// System.out.println("track: " + track.track.toString()) + val master = Cylinder(0.1f, 1.0f, 10) +// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + master.ifMaterial{ +// diffuse = Random.random3DVectorFromRange(0.0f, 1.0f) + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Vector3f(0.8f, 0.7f, 0.7f) + diffuse = Vector3f(0.05f, 0f, 0f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + mInstanced.name = "TrackID-${track.trackId}" +// mInstanced.instancedProperties["ModelMatrix"] = { master.spatial().world } + mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } + addNode(mInstanced) + + + + var cnt = 0 + val a = rng.nextFloat() + val b = rng.nextFloat() + track.track.windowed(2,1).forEach { pair -> + cnt = cnt + 1 + val element = mInstanced.addInstance() + element.name ="EdgeID-$cnt" + element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} + element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) + mInstanced.instances.add(element) + + } + + } + + } + /** * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud @@ -1528,21 +1652,21 @@ class SciView : SceneryBase, CalibratedRealInterval { } renderer!!.pushMode = false // we need to force reloading the renderer as the HMD might require device or instance extensions -// if (renderer is VulkanRenderer && hmdAdded) { -// replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) -// (renderer as VulkanRenderer).toggleVR() -// while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { -// logger.debug("Waiting for renderer reinitialisation") -// try { -// Thread.sleep(200) -// } catch (e: InterruptedException) { -// e.printStackTrace() -// } -// } -// } else { -// renderer!!.toggleVR() -// } - renderer!!.toggleVR() + if (renderer is VulkanRenderer && hmdAdded) { + replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) + (renderer as VulkanRenderer).toggleVR() + while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { + logger.debug("Waiting for renderer reinitialisation") + try { + Thread.sleep(200) + } catch (e: InterruptedException) { + e.printStackTrace() + } + } + } else { + renderer!!.toggleVR() + } +// renderer!!.toggleVR() } /** diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt new file mode 100644 index 00000000..2f18c129 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt @@ -0,0 +1,593 @@ +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.numerics.OpenSimplexNoise +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class ControllerTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.camera!!.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogs.children.removeAt(hedgehogs.children.size-1) + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}") + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } else { + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position + + + while(true) + { + val p = Vector3f(0f,0f,-1f) + referenceTarget.position = p + referenceTarget.visible = true + val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() +// val direction = cam.headOrientation.transform(Vector3f(0.0f,0.0f,-1.0f)) + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + System.out.println("tracking!!!!!!!!!!") + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + //referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + + + } // bind calibration start to menu key on controller + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = true + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) +// System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + var lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = if(lastHedgehog == null) { + val m = Cylinder(3f, 1.0f, 10) + m.ifMaterial { + ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + roughness = 1.0f + metallic = 0.0f + cullingMode = Material.CullingMode.None + } + m.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(m) + mInstanced + } else { + null + } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + if(master != null) { + val element = master.addInstance() + element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + element.parent = volume + master.instances.add(element) + } + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + master?.let { volume.addChild(it) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index dd332126..3e011a66 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -87,7 +87,7 @@ class EyeTrackingDemo: Command{ lateinit var sessionDirectory: Path val hedgehogs = Mesh() - val hedgehogsInstanced = InstancedNode(hedgehogs) + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } var hedgehogVisibility = HedgehogVisibility.Hidden @@ -112,8 +112,6 @@ class EyeTrackingDemo: Command{ override fun run() { - volume = sciview.find("volume") as Volume - volume.visible = false sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" @@ -146,6 +144,9 @@ class EyeTrackingDemo: Command{ shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) sciview.addChild(shell) + volume = sciview.find("volume") as Volume + volume.visible = false + val bb = BoundingGrid() bb.node = volume bb.visible = false @@ -241,16 +242,16 @@ class EyeTrackingDemo: Command{ if(hedgehogs.visible) { if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog -> - val hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instances.forEach { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint } } } else { hedgehogs.children.forEach { hedgehog -> - val hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instances.forEach { it.visible = true } + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } } } } @@ -259,7 +260,7 @@ class EyeTrackingDemo: Command{ tracking = false referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.") + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) dumpHedgehog() } } @@ -270,15 +271,15 @@ class EyeTrackingDemo: Command{ } - fun addHedgehog(parent: Node) { + fun addHedgehog() { val hedgehog = Cylinder(0.005f, 1.0f, 16) hedgehog.visible = false // hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, // listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - val hedgehogInstanced = InstancedNode(hedgehog) + var hedgehogInstanced = InstancedNode(hedgehog) hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - parent.addChild(hedgehogInstanced) + hedgehogs.addChild(hedgehogInstanced) } @@ -307,17 +308,17 @@ class EyeTrackingDemo: Command{ HedgehogVisibility.Hidden -> { hedgehogs.visible = false hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden") + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) } HedgehogVisibility.PerTimePoint -> { hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint") + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) } HedgehogVisibility.Visible -> { hedgehogs.visible = true - cam.showMessage("Hedgehogs visible") + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) } } } @@ -333,7 +334,7 @@ class EyeTrackingDemo: Command{ val fasterOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s") + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) } else { volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) @@ -343,7 +344,7 @@ class EyeTrackingDemo: Command{ val slowerOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s") + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) } else { volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) @@ -363,7 +364,7 @@ class EyeTrackingDemo: Command{ val deleteLastHedgehog = ConfirmableClickBehaviour( armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.", + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), duration = timeout.toInt()) @@ -383,7 +384,7 @@ class EyeTrackingDemo: Command{ hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") hedgehogFileWriter.close() - cam.showMessage("Last track deleted.", + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), duration = 1000) @@ -401,7 +402,7 @@ class EyeTrackingDemo: Command{ val cellDivision = ClickBehaviour { _, _ -> cam.showMessage("Adding cell division", duration = 1000) dumpHedgehog() - addHedgehog(hedgehogs) + addHedgehog() } hmd.addBehaviour("skip_to_next", nextTimepoint) @@ -438,15 +439,15 @@ class EyeTrackingDemo: Command{ pupilTracker.gazeConfidenceThreshold = confidenceThreshold if (!pupilTracker.isCalibrated) { pupilTracker.onCalibrationInProgress = { - cam.showMessage("Crunching equations ...", messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) + cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) } pupilTracker.onCalibrationFailed = { - cam.showMessage("Calibration failed.", messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) + cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) } pupilTracker.onCalibrationSuccess = { - cam.showMessage("Calibration succeeded!", messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) + cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) // cam.children.find { it.name == "debugBoard" }?.visible = true for (i in 0 until 20) { @@ -462,12 +463,12 @@ class EyeTrackingDemo: Command{ val toggleTracking = ClickBehaviour { _, _ -> if (tracking) { referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.") + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) dumpHedgehog() } else { - addHedgehog(hedgehogs) + addHedgehog() referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.") + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) } tracking = !tracking } @@ -533,7 +534,8 @@ class EyeTrackingDemo: Command{ val temp = direction.mul(sphereDist + 2.0f * sphere.radius) val p2 = Vector3f(center).add(temp) - val spine = Cylinder.betweenPoints(p1, p2, 1.0f, segments = 1) + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) spine.visible = true val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) @@ -562,13 +564,8 @@ class EyeTrackingDemo: Command{ val count = samples.filterNotNull().count { it > 0.2f } spine.metadata["spine"] = metadata - val spineInstanced = InstancedNode(spine) - spineInstanced.instancedProperties["ModelMatrix"] = { spine.spatial().world } - spineInstanced.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - - val hedgehogInstanced = InstancedNode(hedgehogs.children.last()) - hedgehogInstanced.instances.add(spineInstanced.addInstance()) - + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } } } } @@ -579,8 +576,8 @@ class EyeTrackingDemo: Command{ * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. * If [hedgehog] is not null, the cell track will not be added to the scene. */ - fun dumpHedgehog(hedgehog: Node? = null) { - val lastHedgehog = hedgehog ?: hedgehogs.children.last() + fun dumpHedgehog() { + var lastHedgehog = hedgehogs.children.last() as InstancedNode val hedgehogId = hedgehogIds.incrementAndGet() val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() @@ -595,8 +592,8 @@ class EyeTrackingDemo: Command{ trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") } - val lastHedgehogInstanced = InstancedNode(lastHedgehog) - val spines = lastHedgehogInstanced.instances.mapNotNull { spine -> + + val spines = lastHedgehog.instances.mapNotNull { spine -> spine.metadata["spine"] as? SpineMetadata } @@ -615,7 +612,7 @@ class EyeTrackingDemo: Command{ if(track == null) { // logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) return } @@ -624,7 +621,7 @@ class EyeTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val master = if(hedgehog == null) { + val master = if(lastHedgehog == null) { val m = Cylinder(3f, 1.0f, 10) m.ifMaterial { ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") @@ -635,7 +632,6 @@ class EyeTrackingDemo: Command{ } m.name = "Track-$hedgehogId" val mInstanced = InstancedNode(m) - mInstanced.instancedProperties["ModelMatrix"] = { m.spatial().world } mInstanced } else { null @@ -649,14 +645,11 @@ class EyeTrackingDemo: Command{ trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> if(master != null) { - val element = Mesh() + val element = master.addInstance() element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) element.parent = volume - val elementInstanced = InstancedNode(element) - elementInstanced.instancedProperties["ModelMatrix"] = { element.spatial().world } - master.instances.add(elementInstanced.addInstance()) + master.instances.add(element) } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") From 0d613f6a087e104dfbb06639fe2d852cd9cd71bf Mon Sep 17 00:00:00 2001 From: ruoshan Date: Mon, 29 Nov 2021 14:21:17 +0100 Subject: [PATCH 07/55] add test without VR headset --- src/main/kotlin/sc/iview/SciView.kt | 2 +- .../demo/advanced/ControllerTrackingDemo.kt | 86 +++- .../sc/iview/commands/demo/advanced/Test.kt | 464 ++++++++++++++++++ 3 files changed, 532 insertions(+), 20 deletions(-) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 84a177ce..858ddd51 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -722,7 +722,7 @@ class SciView : SceneryBase, CalibratedRealInterval { { val v = Volume.fromPath(source, hub) v.name = "volume" - v.spatial().position = Vector3f(0.0f, 1.0f, 0.0f) + v.spatial().position = Vector3f(0.0f, 5.0f, 0.0f) v.colormap = Colormap.get("jet") v.spatial().scale = Vector3f(10.0f, 10.0f,30.0f) v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt index 2f18c129..ca549e41 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt @@ -78,9 +78,14 @@ class ControllerTrackingDemo: Command{ lateinit var hmd: OpenVRHMD val referenceTarget = Icosphere(0.004f, 2) - val calibrationTarget = Icosphere(0.02f, 2) + //val calibrationTarget = Icosphere(0.02f, 2) + //val TestTarget = Icosphere(0.04f, 2) val laser = Cylinder(0.005f, 0.2f, 10) + lateinit var TestTarget1:Icosphere + lateinit var TestTarget2:Icosphere + lateinit var TestTarget3:Icosphere + lateinit var TestTarget4:Icosphere lateinit var sessionId: String lateinit var sessionDirectory: Path @@ -110,6 +115,32 @@ class ControllerTrackingDemo: Command{ var volumeScaleFactor = 1.0f override fun run() { +// center position: (-1.217E-2 2.478E+0 3.981E+0)pointWord: ( 1.650E-1 2.277E+0 3.012E+0) +// sphere.origin: ( 1.500E+0 -3.000E-1 -3.050E-1)p2 position( 1.826E+0 3.938E-1 -6.069E+0) + TestTarget1= Icosphere(0.08f, 2) + TestTarget1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + TestTarget1.spatial().position = Vector3f(-0.1059f, 2.394f, 4.086f)//Vector3f(1.858f,1.7f,2.432f)// + TestTarget1.visible = true + sciview.addChild(TestTarget1) + + TestTarget2= Icosphere(0.08f, 2) + TestTarget2.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + TestTarget2.spatial().position = Vector3f(-0.0318f, 2.438f, 4.084f)//Vector3f(1.858f,1.7f,2.432f)// + TestTarget2.visible = true + sciview.addChild(TestTarget2) + +// TestTarget3= Icosphere(0.08f, 2) +// TestTarget3.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} +// TestTarget3.spatial().position = Vector3f(1.5f, -0.3f, -0.3050f)//Vector3f(1.858f,1.7f,2.432f)// +// TestTarget3.visible = true +// sciview.addChild(TestTarget3) +// + TestTarget4= Icosphere(0.8f, 2) + TestTarget4.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + TestTarget4.spatial().position = Vector3f(1.826f, 0.3938f, -6.069f)//( 1.826E+0 3.938E-1 -6.069E+0) + TestTarget4.visible = true + sciview.addChild(TestTarget4) + sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -124,17 +155,18 @@ class ControllerTrackingDemo: Command{ } sciview.camera!!.addChild(referenceTarget) - calibrationTarget.visible = false - calibrationTarget.material { - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.camera!!.addChild(calibrationTarget) +// calibrationTarget.visible = false +// calibrationTarget.material { +// roughness = 1.0f +// metallic = 0.0f +// diffuse = Vector3f(1.0f, 1.0f, 1.0f)} +// sciview.camera!!.addChild(calibrationTarget) laser.visible = false laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} sciview.addChild(laser) + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) shell.ifMaterial{ cullingMode = Material.CullingMode.Front @@ -144,7 +176,7 @@ class ControllerTrackingDemo: Command{ sciview.addChild(shell) volume = sciview.find("volume") as Volume - volume.visible = false + //volume.visible = false val bb = BoundingGrid() bb.node = volume @@ -361,7 +393,7 @@ class ControllerTrackingDemo: Command{ } else { PlaybackDirection.Forward } - cam.showMessage("Playing: ${direction}") + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) }) val cellDivision = ClickBehaviour { _, _ -> @@ -422,21 +454,31 @@ class ControllerTrackingDemo: Command{ while(true) { - val p = Vector3f(0f,0f,-1f) - referenceTarget.position = p + val p = Vector3f(0.1f,0f,-2.0f) + referenceTarget.spatial().position = p referenceTarget.visible = true - val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() + //val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() + laser.spatial().orientBetweenPoints(headCenter,pointWorld) + laser.visible = true + + +// TestTarget.visible = true + TestTarget1.ifSpatial { position = headCenter} + TestTarget2.ifSpatial { position = pointWorld} + +// print("center position: " + headCenter.toString()) +// println("pointWord: " + pointWorld.toString()) + val direction = (pointWorld - headCenter).normalize() // val direction = cam.headOrientation.transform(Vector3f(0.0f,0.0f,-1.0f)) if (tracking) { // log.info("Starting spine from $headCenter to $pointWorld") - System.out.println("tracking!!!!!!!!!!") + //System.out.println("tracking!!!!!!!!!!") addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) } } - //referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units + } // bind calibration start to menu key on controller @@ -453,17 +495,23 @@ class ControllerTrackingDemo: Command{ val temp = direction.mul(sphereDist + 2.0f * sphere.radius) val p2 = Vector3f(center).add(temp) +// print("sphere.origin: " + sphere.origin.toString()) +// println("p2 position" + p2.toString()) + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() spine.spatial().orientBetweenPoints(p1, p2, true, true) spine.visible = true val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) -// System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { +// System.out.println(intersection); // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null +// System.out.println("localEntry:"+ localEntry.toString()) +// System.out.println("localExit:" + localExit.toString()) if (samples != null && localDirection != null) { val metadata = SpineMetadata( @@ -480,7 +528,7 @@ class ControllerTrackingDemo: Command{ confidence, samples.map { it ?: 0.0f } ) - val count = samples.filterNotNull().count { it > 0.2f } + val count = samples.filterNotNull().count { it > 0.02f } spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt new file mode 100644 index 00000000..4ef3825e --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -0,0 +1,464 @@ +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.numerics.OpenSimplexNoise +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Test without VR and eyetracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class Test: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + //val calibrationTarget = Icosphere(0.02f, 2) + val TestTarget = Icosphere(0.1f, 2) + + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var point1:Icosphere + lateinit var point2:Icosphere + + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Backward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + sciview.addChild(TestTarget) + TestTarget.visible = false + + +// sciview.toggleVRRendering() +// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + point1 = Icosphere(0.1f, 2) + point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) + point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + sciview.addChild(point1) + + point2 = Icosphere(0.1f, 2) + point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) + point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} + sciview.addChild(point2) + + val connector = Cylinder.betweenPoints(point1.position, point2.position) + connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(connector) + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + tracking = true + //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position + + while(true) + { + val p = Vector3f(0f,0f,-1f) + referenceTarget.position = p + referenceTarget.visible = true + val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() + + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + //System.out.println("tracking!!!!!!!!!!") +// println("direction:"+ direction.toString()) + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + //referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + + + } // bind calibration start to menu key on controller + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + + + + val p2 = Vector3f(center).add(temp) + + +// print("center position: " + p1.toString()) +// print("p2 position" + p2.toString()) + + TestTarget.visible = true + TestTarget.ifSpatial { position = p2} + + +// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() +// spine.spatial().orientBetweenPoints(p1, p2, true, true) +// spine.visible = true + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) +// System.out.println("localEntry:"+ localEntry.toString()) +// System.out.println("localExit:" + localExit.toString()) + + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + if(count >0 ) + { + println("count of samples: "+ count.toString()) + } + +// spine.metadata["spine"] = metadata +// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + var lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = if(lastHedgehog == null) { + val m = Cylinder(3f, 1.0f, 10) + m.ifMaterial { + ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + roughness = 1.0f + metallic = 0.0f + cullingMode = Material.CullingMode.None + } + m.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(m) + mInstanced + } else { + null + } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + if(master != null) { + val element = master.addInstance() + element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + element.parent = volume + master.instances.add(element) + } + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + master?.let { volume.addChild(it) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file From 3ddfcfced145fab97a4a3ae43084494fa4d1631b Mon Sep 17 00:00:00 2001 From: ruoshan Date: Fri, 3 Dec 2021 19:32:09 +0100 Subject: [PATCH 08/55] a working version of utilizing VRHeadset for cell tracking - part of sandbox function --- build.gradle.kts | 2 +- src/main/kotlin/sc/iview/SciView.kt | 23 +- .../commands/demo/advanced/EyeTrackingDemo.kt | 2 +- .../demo/advanced/HedgehogAnalysis.kt | 62 ++--- .../sc/iview/commands/demo/advanced/Test.kt | 70 +++++- ...ackingDemo.kt => VRHeadSetTrackingDemo.kt} | 215 ++++++------------ 6 files changed, 177 insertions(+), 197 deletions(-) rename src/main/kotlin/sc/iview/commands/demo/advanced/{ControllerTrackingDemo.kt => VRHeadSetTrackingDemo.kt} (74%) diff --git a/build.gradle.kts b/build.gradle.kts index b3c5304d..53bbdd70 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,7 +36,7 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "37fc0836c8" + val sceneryVersion = "84dfb997fc" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 858ddd51..cc1b9a69 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -722,9 +722,9 @@ class SciView : SceneryBase, CalibratedRealInterval { { val v = Volume.fromPath(source, hub) v.name = "volume" - v.spatial().position = Vector3f(0.0f, 5.0f, 0.0f) + v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) v.colormap = Colormap.get("jet") - v.spatial().scale = Vector3f(10.0f, 10.0f,30.0f) + v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) v.metadata["animating"] = true v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) @@ -775,21 +775,15 @@ class SciView : SceneryBase, CalibratedRealInterval { nodeScore, edgeScore ) -// System.out.println(currentPointInTrack) if(lastTrackId != trackId) { -// Thread.sleep(2000) lastTrackId = trackId -// System.out.println("trackId: "+trackId.toString()) val sortedTrack = track.sortedBy { it.t } tracks.add(Track(sortedTrack, trackId)) -// System.out.println(sortedTrack) track.clear() } - track.add(currentPointInTrack) - } val timeCost = measureTimeMillis { addTracks(tracks) @@ -797,8 +791,6 @@ class SciView : SceneryBase, CalibratedRealInterval { println("time: $timeCost") } - - fun addTracks(tracks: ArrayList) { val rng = Random(17) @@ -806,18 +798,14 @@ class SciView : SceneryBase, CalibratedRealInterval { { if(track.trackId > 10) { -// continue + continue } -// Thread.sleep(2000) System.out.println("add track: "+ track.trackId.toString() ) -// System.out.println("track: " + track.track.toString()) val master = Cylinder(0.1f, 1.0f, 10) // master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) master.ifMaterial{ -// diffuse = Random.random3DVectorFromRange(0.0f, 1.0f) ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Vector3f(0.8f, 0.7f, 0.7f) diffuse = Vector3f(0.05f, 0f, 0f) metallic = 0.01f roughness = 0.5f @@ -825,12 +813,9 @@ class SciView : SceneryBase, CalibratedRealInterval { val mInstanced = InstancedNode(master) mInstanced.name = "TrackID-${track.trackId}" -// mInstanced.instancedProperties["ModelMatrix"] = { master.spatial().world } mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } addNode(mInstanced) - - var cnt = 0 val a = rng.nextFloat() val b = rng.nextFloat() @@ -840,7 +825,7 @@ class SciView : SceneryBase, CalibratedRealInterval { element.name ="EdgeID-$cnt" element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) - mInstanced.instances.add(element) + //mInstanced.instances.add(element) } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 3e011a66..8c3991ed 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -68,7 +68,7 @@ import graphics.scenery.primitives.TextBoard menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize Eye Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) + Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) class EyeTrackingDemo: Command{ @Parameter private lateinit var sciview: SciView diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 7bfb4d15..8a2d432e 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -114,6 +114,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val startingThreshold = 0.02f val localMaxThreshold = 0.01f val zscoreThreshold = 2.0f + val removeTooFarThreshold = 5.0f if(timepoints.isEmpty()) { return null @@ -166,8 +167,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix System.out.println("initial:"+initial) var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> -// System.out.println(time) - +// System.out.println("time: ${time}") +// println("vs: ${vs}") val distances = vs .filter { it.value > localMaxThreshold } .map { vertex -> @@ -176,11 +177,13 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix vertex to distance } .sortedBy { it.second } -// logger.info("distances - ${distances}") -// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") - +// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) +// { +// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") +// } +// val closest = distances.firstOrNull()?.first - if(closest != null) { + if(closest != null && distances.firstOrNull()?.second!! >0) { current.next = closest closest.previous = current current = closest @@ -190,32 +193,36 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } }.toMutableList() - - - var check = shortestPath - - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) val beforeCount = shortestPath.size System.out.println("before short path:"+ shortestPath.size) - while (shortestPath.any { it.distance() >= 2.0f * avgPathLength }) { - shortestPath.filter { it.distance() >= 2.0f * avgPathLength }.forEach { it.drop() } - shortestPath = shortestPath.filter { it.distance() < 2.0f * avgPathLength }.toMutableList() - avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - } + while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { + shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) +// println("check which one is removed") +// shortestPath.forEach { +// if(it.distance() >= removeTooFarThreshold * avgPathLength) +// { +// println("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) +// } +// } } +// + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } // logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") while(remaining > 0) { @@ -228,8 +235,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() -// logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - + //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } @@ -239,9 +245,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix it.getOrNull(1)?.next = it.getOrNull(2) it.getOrNull(2)?.previous = it.getOrNull(1) } -// avgPathLength = shortestPath.map { it.distance() }.average().toFloat() -// stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + //logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") } val afterCount = shortestPath.size @@ -400,7 +404,7 @@ fun main(args: Array) { // return // } - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone-2021-08-10 16.17.40\\Hedgehog_1_2021-08-10 16.21.35.csv") + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") // val analysis = HedgehogAnalysis.fromIncompleteCSV(file) val analysis = HedgehogAnalysis.fromCSV(file) val results = analysis.run() diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index 4ef3825e..d758b6c8 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -63,12 +63,13 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import sc.iview.commands.demo.animation.ParticleDemo @Plugin(type = Command::class, menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Test without VR and eyetracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) + Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) class Test: Command{ @Parameter private lateinit var sciview: SciView @@ -157,9 +158,24 @@ class Test: Command{ point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} sciview.addChild(point2) - val connector = Cylinder.betweenPoints(point1.position, point2.position) - connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(connector) + var ccc= 0; + while(true) + { + ccc = ccc + 1 + if(ccc == 1000) + { + break + } + var w = ccc*0.01f + point1.position = point1.position.add(Vector3f(w,w,w)) + point2.position = point2.position.add(Vector3f(w,w,w)) + val connector = Cylinder(0.1f, 1.0f, 16) + connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) +// val connector = Cylinder.betweenPoints(point1.position, point2.position) + connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(connector) + + } val bb = BoundingGrid() bb.node = volume @@ -270,7 +286,7 @@ class Test: Command{ tracking = true //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position - while(true) + if(true) { val p = Vector3f(0f,0f,-1f) referenceTarget.position = p @@ -285,6 +301,7 @@ class Test: Command{ //System.out.println("tracking!!!!!!!!!!") // println("direction:"+ direction.toString()) addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + showTrack() } } //referenceTarget.visible = true @@ -294,6 +311,49 @@ class Test: Command{ } // bind calibration start to menu key on controller } + + private fun showTrack() + { + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-30 14.09.43\\Hedgehog_1_2021-11-30 14.10.21.csv") + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,volume.world,Vector3f(volume.getDimensions())) + val track = analysis.run() + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + sciview.addNode(mInstanced) + + val volumeDimensions = volume.getDimensions() + + if(track == null) + { + return + } + track.points.windowed(2, 1).forEach { pair -> + + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + mInstanced.instances.add(element) + +// val pp = Icosphere(0.1f, 1) +// pp.spatial().position = p0w +// pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) +// sciview.addChild(pp) + } + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { val cam = sciview.camera as? DetachedHeadCamera ?: return val sphere = volume.boundingBox?.getBoundingSphere() ?: return diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt similarity index 74% rename from src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index ca549e41..75b8304f 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/ControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -11,9 +11,6 @@ import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew -import graphics.scenery.numerics.OpenSimplexNoise import graphics.scenery.numerics.Random import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects @@ -63,13 +60,14 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import sc.iview.commands.demo.animation.ParticleDemo @Plugin(type = Command::class, menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Controller for Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class ControllerTrackingDemo: Command{ + Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRHeadSetTrackingDemo: Command{ @Parameter private lateinit var sciview: SciView @@ -77,28 +75,18 @@ class ControllerTrackingDemo: Command{ private lateinit var log: LogService lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) - //val calibrationTarget = Icosphere(0.02f, 2) - //val TestTarget = Icosphere(0.04f, 2) - val laser = Cylinder(0.005f, 0.2f, 10) - - lateinit var TestTarget1:Icosphere - lateinit var TestTarget2:Icosphere - lateinit var TestTarget3:Icosphere - lateinit var TestTarget4:Icosphere + val referenceTarget = Icosphere(0.04f, 2) lateinit var sessionId: String lateinit var sessionDirectory: Path - val hedgehogs = Mesh() + var hedgehogsList = mutableListOf() enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } var hedgehogVisibility = HedgehogVisibility.Hidden lateinit var volume: Volume - val confidenceThreshold = 0.60f - enum class PlaybackDirection { Forward, Backward @@ -115,32 +103,6 @@ class ControllerTrackingDemo: Command{ var volumeScaleFactor = 1.0f override fun run() { -// center position: (-1.217E-2 2.478E+0 3.981E+0)pointWord: ( 1.650E-1 2.277E+0 3.012E+0) -// sphere.origin: ( 1.500E+0 -3.000E-1 -3.050E-1)p2 position( 1.826E+0 3.938E-1 -6.069E+0) - TestTarget1= Icosphere(0.08f, 2) - TestTarget1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - TestTarget1.spatial().position = Vector3f(-0.1059f, 2.394f, 4.086f)//Vector3f(1.858f,1.7f,2.432f)// - TestTarget1.visible = true - sciview.addChild(TestTarget1) - - TestTarget2= Icosphere(0.08f, 2) - TestTarget2.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - TestTarget2.spatial().position = Vector3f(-0.0318f, 2.438f, 4.084f)//Vector3f(1.858f,1.7f,2.432f)// - TestTarget2.visible = true - sciview.addChild(TestTarget2) - -// TestTarget3= Icosphere(0.08f, 2) -// TestTarget3.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} -// TestTarget3.spatial().position = Vector3f(1.5f, -0.3f, -0.3050f)//Vector3f(1.858f,1.7f,2.432f)// -// TestTarget3.visible = true -// sciview.addChild(TestTarget3) -// - TestTarget4= Icosphere(0.8f, 2) - TestTarget4.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - TestTarget4.spatial().position = Vector3f(1.826f, 0.3938f, -6.069f)//( 1.826E+0 3.938E-1 -6.069E+0) - TestTarget4.visible = true - sciview.addChild(TestTarget4) - sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -155,17 +117,6 @@ class ControllerTrackingDemo: Command{ } sciview.camera!!.addChild(referenceTarget) -// calibrationTarget.visible = false -// calibrationTarget.material { -// roughness = 1.0f -// metallic = 0.0f -// diffuse = Vector3f(1.0f, 1.0f, 1.0f)} -// sciview.camera!!.addChild(calibrationTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) shell.ifMaterial{ @@ -182,12 +133,6 @@ class ControllerTrackingDemo: Command{ bb.node = volume bb.visible = false - sciview.addChild(hedgehogs) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - val debugBoard = TextBoard() debugBoard.name = "debugBoard" @@ -237,25 +182,18 @@ class ControllerTrackingDemo: Command{ val newTimepoint = volume.viewerState.currentTimepoint - if(hedgehogs.visible) { + if(hedgehogsList.size>0) { if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode + hedgehogsList.forEach { hedgehog-> hedgehog.instances.forEach { it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint } } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } } } if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) dumpHedgehog() @@ -269,14 +207,18 @@ class ControllerTrackingDemo: Command{ } fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) } @@ -303,18 +245,23 @@ class ControllerTrackingDemo: Command{ when(hedgehogVisibility) { HedgehogVisibility.Hidden -> { - hedgehogs.visible = false - hedgehogs.runRecursive { it.visible = false } + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) } HedgehogVisibility.PerTimePoint -> { - hedgehogs.visible = true cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) } HedgehogVisibility.Visible -> { - hedgehogs.visible = true + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) } } @@ -368,7 +315,7 @@ class ControllerTrackingDemo: Command{ }, confirmAction = { - hedgehogs.children.removeAt(hedgehogs.children.size-1) + hedgehogsList = hedgehogsList.dropLast(1) as MutableList volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> volume.removeChild(lastTrack) } @@ -435,13 +382,16 @@ class ControllerTrackingDemo: Command{ if (tracking) { referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) } else { addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true } - tracking = !tracking } hmd.addBehaviour("toggle_tracking", toggleTracking) hmd.addKeyBinding("toggle_tracking", keybindingTracking) @@ -449,69 +399,48 @@ class ControllerTrackingDemo: Command{ volume.visible = true volume.runRecursive { it.visible = true } playing = true - //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position while(true) { - val p = Vector3f(0.1f,0f,-2.0f) - referenceTarget.spatial().position = p - referenceTarget.visible = true - //val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) - val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() - val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() - laser.spatial().orientBetweenPoints(headCenter,pointWorld) - laser.visible = true + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() -// TestTarget.visible = true - TestTarget1.ifSpatial { position = headCenter} - TestTarget2.ifSpatial { position = pointWorld} + referenceTarget.visible = true + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-2f) } -// print("center position: " + headCenter.toString()) -// println("pointWord: " + pointWorld.toString()) val direction = (pointWorld - headCenter).normalize() -// val direction = cam.headOrientation.transform(Vector3f(0.0f,0.0f,-1.0f)) if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - //System.out.println("tracking!!!!!!!!!!") addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) } } - - - - } // bind calibration start to menu key on controller + } } fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { val cam = sciview.camera as? DetachedHeadCamera ?: return val sphere = volume.boundingBox?.getBoundingSphere() ?: return - val sphereDirection = sphere.origin.minus(center) + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) val p2 = Vector3f(center).add(temp) -// print("sphere.origin: " + sphere.origin.toString()) -// println("p2 position" + p2.toString()) - - val spine = (hedgehogs.children.last() as InstancedNode).addInstance() - spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = true + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) if(intersection is MaybeIntersects.Intersection) { -// System.out.println(intersection); // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null -// System.out.println("localEntry:"+ localEntry.toString()) -// System.out.println("localExit:" + localExit.toString()) if (samples != null && localDirection != null) { val metadata = SpineMetadata( @@ -529,10 +458,10 @@ class ControllerTrackingDemo: Command{ samples.map { it ?: 0.0f } ) val count = samples.filterNotNull().count { it > 0.02f } - + //println("cnt: " + count.toString()) spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } } } } @@ -544,7 +473,9 @@ class ControllerTrackingDemo: Command{ * If [hedgehog] is not null, the cell track will not be added to the scene. */ fun dumpHedgehog() { - var lastHedgehog = hedgehogs.children.last() as InstancedNode + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") val hedgehogId = hedgehogIds.incrementAndGet() val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() @@ -588,42 +519,42 @@ class ControllerTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val master = if(lastHedgehog == null) { - val m = Cylinder(3f, 1.0f, 10) - m.ifMaterial { - ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - roughness = 1.0f - metallic = 0.0f - cullingMode = Material.CullingMode.None - } - m.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(m) - mInstanced - } else { - null + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) val parentId = 0 val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) trackFileWriter.newLine() trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> - if(master != null) { - val element = master.addInstance() - element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) - element.parent = volume - master.instances.add(element) - } + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + //mInstanced.instances.add(element) + val pp = Icosphere(0.01f, 1) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addChild(pp) + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } - - master?.let { volume.addChild(it) } - trackFileWriter.close() } From efe9902ab7ca9613af50e37dc1f0607cacef13cd Mon Sep 17 00:00:00 2001 From: ruoshan Date: Fri, 17 Dec 2021 01:13:54 +0100 Subject: [PATCH 09/55] add function of tracking with controllers --- .../demo/advanced/TrackingDragBehaviour.kt | 17 + .../demo/advanced/VRControllerTrackingDemo.kt | 636 ++++++++++++++++++ .../demo/advanced/VRHeadSetTrackingDemo.kt | 8 +- 3 files changed, 657 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt new file mode 100644 index 00000000..ce7b42b9 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingDragBehaviour.kt @@ -0,0 +1,17 @@ +package sc.iview.commands.demo.advanced + +import org.scijava.ui.behaviour.DragBehaviour + +class TrackingDragBehaviour():DragBehaviour{ + override fun init(x: Int, y: Int) { + TODO("Not yet implemented") + } + + override fun drag(x: Int, y: Int) { + TODO("Not yet implemented") + } + + override fun end(x: Int, y: Int) { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt new file mode 100644 index 00000000..959a32b5 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -0,0 +1,636 @@ +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import org.scijava.ui.behaviour.DragBehaviour +import sc.iview.commands.demo.animation.ParticleDemo + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRControllerTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.04f, 2) + val testTarget1 = Icosphere(0.01f, 2) + val testTarget2 = Icosphere(0.04f, 2) + val laser = Cylinder(0.0025f, 1f, 20) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var rightController: TrackedDevice + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) + laser.material().metallic = 0.0f + laser.material().roughness = 1.0f + laser.visible = false + sciview.addNode(laser) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(referenceTarget) + + testTarget1.visible = false + testTarget1.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget1) + + + testTarget2.visible = false + testTarget2.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget2) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume +// volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") +// if(device.role == TrackerRole.RightHand) { +// rightController = device +// log.info("rightController is found, its location is in ${rightController.position}") +// } +// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + println("setupControllerforTracking") + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + + println("test") + + while(true) + { + if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) + { + println("null") + continue + } + else + { + rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! + + if (rightController.model?.spatialOrNull() == null) { + println("spatial null") + } + else + { + val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() + val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + +// println(headCenter.toString()) +// println(pointWorld.toString()) + testTarget1.visible = true + testTarget1.ifSpatial { position = headCenter} + + testTarget2.visible = true + testTarget2.ifSpatial { position = pointWorld} + + laser.visible = true + laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = pointWorld} + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + + } + + } + + } + + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.02f } + //println("cnt: " + count.toString()) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + //mInstanced.instances.add(element) + val pp = Icosphere(0.01f, 1) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index 75b8304f..903724b2 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -127,7 +127,7 @@ class VRHeadSetTrackingDemo: Command{ sciview.addChild(shell) volume = sciview.find("volume") as Volume - //volume.visible = false +// volume.visible = false val bb = BoundingGrid() bb.node = volume @@ -316,9 +316,9 @@ class VRHeadSetTrackingDemo: Command{ }, confirmAction = { hedgehogsList = hedgehogsList.dropLast(1) as MutableList - volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> - volume.removeChild(lastTrack) - } +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } val hedgehogId = hedgehogIds.get() val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() From 73705198af1b3cbdb7e8277744c61dc50a4acb5b Mon Sep 17 00:00:00 2001 From: ruoshan Date: Thu, 19 May 2022 12:44:08 +0200 Subject: [PATCH 10/55] replace numerictype with realtype --- build.gradle.kts | 3 +- pom.xml | 4 +- settings.gradle.kts | 2 +- src/main/kotlin/sc/iview/SciView.kt | 20 +- .../demo/advanced/HedgehogAnalysis.kt | 23 +- .../sc/iview/commands/demo/advanced/Test.kt | 324 +++++++++--------- .../demo/advanced/VRControllerTrackingDemo.kt | 32 +- .../demo/advanced/VRHeadSetTrackingDemo.kt | 14 +- 8 files changed, 216 insertions(+), 206 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 53bbdd70..cd6facbd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,7 +36,8 @@ dependencies { annotationProcessor(sciJava.common) kapt(sciJava.common) - val sceneryVersion = "84dfb997fc" + val sceneryVersion = "71149792ec" + //val sceneryVersion = "84dfb997fc" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab // if not, uncomment this only to trigger it diff --git a/pom.xml b/pom.xml index 75a48644..5cc17386 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ ${scijava.jvm.version} [1.8.0-101,) - 5de0b1e + 84dfb997fc 2.0.3 0.8.1 @@ -354,7 +354,7 @@ - src/test/tests + src/test org.apache.maven.plugins diff --git a/settings.gradle.kts b/settings.gradle.kts index 02f05118..d7739faa 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,7 +14,7 @@ rootProject.name = "sciview" gradle.rootProject { group = "graphics.scenery" - version = "0.2.0-beta-9-SNAPSHOT" + version = "0.2.0-beta-9-SNAPSHOT-test-4" description = "Scenery-backed 3D visualization package for ImageJ." } diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index cc1b9a69..72779667 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -39,15 +39,12 @@ import bdv.viewer.Source import bdv.viewer.SourceAndConverter import graphics.scenery.* import graphics.scenery.Scene.RaycastResult -import graphics.scenery.attribute.material.Material import graphics.scenery.backends.Renderer import graphics.scenery.backends.opengl.OpenGLRenderer import graphics.scenery.backends.vulkan.VulkanRenderer import graphics.scenery.controls.InputHandler import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedStereoGlasses import graphics.scenery.controls.TrackerInput -import graphics.scenery.numerics.Random import graphics.scenery.primitives.* import graphics.scenery.proteins.Protein import graphics.scenery.proteins.RibbonDiagram @@ -80,7 +77,6 @@ import net.imglib2.* import net.imglib2.display.ColorTable import net.imglib2.img.Img import net.imglib2.img.array.ArrayImgs -import net.imglib2.img.display.imagej.ImageJFunctions import net.imglib2.realtransform.AffineTransform3D import net.imglib2.type.numeric.ARGBType import net.imglib2.type.numeric.NumericType @@ -89,6 +85,7 @@ import net.imglib2.type.numeric.integer.UnsignedByteType import net.imglib2.view.Views import org.joml.Quaternionf import org.joml.Vector3f +import org.joml.Vector4f import org.scijava.Context import org.scijava.`object`.ObjectService import org.scijava.display.Display @@ -104,11 +101,11 @@ import org.scijava.thread.ThreadService import org.scijava.util.ColorRGB import org.scijava.util.Colors import org.scijava.util.VersionUtils +import sc.iview.commands.demo.animation.ParticleDemo import sc.iview.event.NodeActivatedEvent import sc.iview.event.NodeAddedEvent import sc.iview.event.NodeChangedEvent import sc.iview.event.NodeRemovedEvent -import sc.iview.node.Line3D import sc.iview.process.MeshConverter import sc.iview.ui.CustomPropertyUI import sc.iview.ui.MainWindow @@ -127,12 +124,9 @@ import java.util.function.Consumer import java.util.function.Function import java.util.function.Predicate import java.util.stream.Collectors -import kotlin.collections.ArrayList import kotlin.math.cos import kotlin.math.sin import kotlin.system.measureTimeMillis -import org.joml.Vector4f -import sc.iview.commands.demo.animation.ParticleDemo /** * Main SciView class. @@ -729,7 +723,15 @@ class SciView : SceneryBase, CalibratedRealInterval { v.metadata["animating"] = true v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) v.visible = true + + v.spatial().wantsComposeModel = true + v.spatial().updateWorld(true) + System.out.println("v.model: " + v.model) addChild(v) + System.out.println("v.getDimensions: "+ v.getDimensions()) + + System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) + System.out.println("v.world.matrix: " + v.spatial().world) } data class PointInTrack( @@ -1465,7 +1467,7 @@ class SciView : SceneryBase, CalibratedRealInterval { */ @JvmOverloads @Suppress("UNCHECKED_CAST") - fun > addVolume(sources: List>, + fun > addVolume(sources: List>, converterSetups: ArrayList, numTimepoints: Int, name: String = "Volume", diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 8a2d432e..18e0c3f4 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -136,7 +136,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val candidates = timepoints.map { tp -> val vs = tp.value.mapIndexedNotNull { i, spine -> val maxIndices = localMaxima(spine.samples.filterNotNull()) - logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + //logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") if(maxIndices.isNotEmpty()) { @@ -144,7 +144,12 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix map { index -> // logger.info(index.toString()) val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) +// println("i: " + i) +// println("position: " + position) +// println("dimension: "+ dimension) +// println("localToWorld: "+ localToWorld) val worldPosition = localToWorld.transform((Vector3f(position).mul(dimension)).xyzw()).xyz() +// println("world position: "+ worldPosition) SpineGraphVertex(tp.key, position, worldPosition, @@ -165,6 +170,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // get the initial vertex, this one is assumed to always be in front, and have a local max val initial = candidates.first().filter{it.value>startingThreshold}.first() System.out.println("initial:"+initial) + System.out.println("candidates number: "+ candidates.size) var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> // System.out.println("time: ${time}") @@ -174,9 +180,13 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix .map { vertex -> val t = current.worldPosition - vertex.worldPosition val distance = t.length() +// println("current worldposition:"+ current.worldPosition) +// println("vertex.worldposition"+vertex.worldPosition) vertex to distance } .sortedBy { it.second } + //println("distances.size: "+distances.size) + //println("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) // if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) // { // logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") @@ -193,13 +203,16 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } }.toMutableList() + + val beforeCount = shortestPath.size + System.out.println("before short path:"+ shortestPath.size) + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) - val beforeCount = shortestPath.size - System.out.println("before short path:"+ shortestPath.size) + while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() @@ -224,7 +237,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } -// logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") while(remaining > 0) { val outliers = shortestPath .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index d758b6c8..7ef1b6d6 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -1,69 +1,34 @@ package sc.iview.commands.demo.advanced -import bdv.util.BdvFunctions import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.attribute.material.Material import graphics.scenery.bionictracking.HedgehogAnalysis import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew -import graphics.scenery.numerics.OpenSimplexNoise import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* -import org.scijava.Context import org.scijava.command.Command import org.scijava.command.CommandService +import org.scijava.log.LogService import org.scijava.plugin.Menu import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream import java.io.File -import java.io.FileWriter import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.util.HashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import sc.iview.commands.demo.animation.ParticleDemo @Plugin(type = Command::class, menuRoot = "SciView", @@ -108,7 +73,8 @@ class Test: Command{ @Volatile var tracking = false var playing = false var direction = PlaybackDirection.Backward - var volumesPerSecond = 4 + @Parameter(label = "Volumes per second") + var volumesPerSecond = 1 var skipToNext = false var skipToPrevious = false // var currentVolume = 0 @@ -141,41 +107,41 @@ class Test: Command{ shell.ifMaterial{ cullingMode = Material.CullingMode.Front diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - + shell.name = "shell" shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) sciview.addChild(shell) volume = sciview.find("volume") as Volume volume.visible = false - point1 = Icosphere(0.1f, 2) - point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) - point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - sciview.addChild(point1) - - point2 = Icosphere(0.1f, 2) - point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) - point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} - sciview.addChild(point2) - - var ccc= 0; - while(true) - { - ccc = ccc + 1 - if(ccc == 1000) - { - break - } - var w = ccc*0.01f - point1.position = point1.position.add(Vector3f(w,w,w)) - point2.position = point2.position.add(Vector3f(w,w,w)) - val connector = Cylinder(0.1f, 1.0f, 16) - connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) -// val connector = Cylinder.betweenPoints(point1.position, point2.position) - connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(connector) - - } +// point1 = Icosphere(0.1f, 2) +// point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) +// point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} +// sciview.addChild(point1) +// +// point2 = Icosphere(0.1f, 2) +// point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) +// point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} +// sciview.addChild(point2) +// +// var ccc= 0; +// while(true) +// { +// ccc = ccc + 1 +// if(ccc == 1000) +// { +// break +// } +// var w = ccc*0.01f +// point1.position = point1.position.add(Vector3f(w,w,w)) +// point2.position = point2.position.add(Vector3f(w,w,w)) +// val connector = Cylinder(0.1f, 1.0f, 16) +// connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) +//// val connector = Cylinder.betweenPoints(point1.position, point2.position) +// connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} +// sciview.addChild(connector) +// +// } val bb = BoundingGrid() bb.node = volume @@ -225,7 +191,7 @@ class Test: Command{ } } val newTimepoint = volume.viewerState.currentTimepoint - + //println("timepoint: "+ newTimepoint); if(hedgehogs.visible) { if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { @@ -248,7 +214,7 @@ class Test: Command{ referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() + //dumpHedgehog() } } @@ -288,19 +254,19 @@ class Test: Command{ if(true) { - val p = Vector3f(0f,0f,-1f) - referenceTarget.position = p - referenceTarget.visible = true - val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() - - val direction = (pointWorld - headCenter).normalize() +// val p = Vector3f(0f,0f,-1f) +// referenceTarget.position = p +// referenceTarget.visible = true +// val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) +// val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() +// +// val direction = (pointWorld - headCenter).normalize() if (tracking) { // log.info("Starting spine from $headCenter to $pointWorld") //System.out.println("tracking!!!!!!!!!!") // println("direction:"+ direction.toString()) - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + //addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) showTrack() } } @@ -314,13 +280,27 @@ class Test: Command{ private fun showTrack() { - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-30 14.09.43\\Hedgehog_1_2021-11-30 14.10.21.csv") - val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,volume.world,Vector3f(volume.getDimensions())) + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-03-10 10.19.49\\Hedgehog_2_2022-03-10 10.21.10.csv") + var volumeDimensions = volume.getDimensions() + var selfdefineworlfmatrix = volume.spatial().world + // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) + selfdefineworlfmatrix = Matrix4f( + 0.015f, 0f, 0f, 0f, + 0f, -0.015f, 0f, 0f, + 0f, 0f, 0.045f, 0f, + -5f, 8f, -2f, 1f + ) + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix,Vector3f(volumeDimensions)) + print("volume.getDimensions(): "+ volume.getDimensions()) + print("volume.spatial().world: "+ volume.spatial().world) + print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) + val track = analysis.run() + print("flag1") val master = Cylinder(0.1f, 1.0f, 10) master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - + print("flag2") master.ifMaterial{ ambient = Vector3f(0.1f, 0f, 0f) diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) @@ -331,12 +311,13 @@ class Test: Command{ val mInstanced = InstancedNode(master) sciview.addNode(mInstanced) - val volumeDimensions = volume.getDimensions() + print("flag3") if(track == null) { return } + print("flag4") track.points.windowed(2, 1).forEach { pair -> val element = mInstanced.addInstance() @@ -345,12 +326,13 @@ class Test: Command{ val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - mInstanced.instances.add(element) -// val pp = Icosphere(0.1f, 1) -// pp.spatial().position = p0w -// pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) -// sciview.addChild(pp) + val tp = pair[0].second.timepoint + val pp = Icosphere(0.01f, 1) + pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addNode(pp) } } @@ -426,89 +408,89 @@ class Test: Command{ * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. * If [hedgehog] is not null, the cell track will not be added to the scene. */ - fun dumpHedgehog() { - var lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = if(lastHedgehog == null) { - val m = Cylinder(3f, 1.0f, 10) - m.ifMaterial { - ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - roughness = 1.0f - metallic = 0.0f - cullingMode = Material.CullingMode.None - } - m.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(m) - mInstanced - } else { - null - } - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - if(master != null) { - val element = master.addInstance() - element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) - element.parent = volume - master.instances.add(element) - } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - master?.let { volume.addChild(it) } - - trackFileWriter.close() - } +// fun dumpHedgehog() { +// var lastHedgehog = hedgehogs.children.last() as InstancedNode +// val hedgehogId = hedgehogIds.incrementAndGet() +// +// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() +// val hedgehogFileWriter = hedgehogFile.bufferedWriter() +// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") +// +// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() +// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) +// if(!trackFile.exists()) { +// trackFile.createNewFile() +// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") +// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") +// } +// +// +// val spines = lastHedgehog.instances.mapNotNull { spine -> +// spine.metadata["spine"] as? SpineMetadata +// } +// +// spines.forEach { metadata -> +// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") +// } +// hedgehogFileWriter.close() +// +// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track +// val track = if(existingAnalysis is HedgehogAnalysis.Track) { +// existingAnalysis +// } else { +// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) +// h.run() +// } +// +// if(track == null) { +//// logger.warn("No track returned") +// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) +// return +// } +// +// lastHedgehog.metadata["HedgehogAnalysis"] = track +// lastHedgehog.metadata["Spines"] = spines +// +//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") +// +// val master = if(lastHedgehog == null) { +// val m = Cylinder(3f, 1.0f, 10) +// m.ifMaterial { +// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") +// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) +// roughness = 1.0f +// metallic = 0.0f +// cullingMode = Material.CullingMode.None +// } +// m.name = "Track-$hedgehogId" +// val mInstanced = InstancedNode(m) +// mInstanced +// } else { +// null +// } +// +// val parentId = 0 +// val volumeDimensions = volume.getDimensions() +// +// trackFileWriter.newLine() +// trackFileWriter.newLine() +// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") +// track.points.windowed(2, 1).forEach { pair -> +// if(master != null) { +// val element = master.addInstance() +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) +// element.parent = volume +// master.instances.add(element) +// } +// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product +// val tp = pair[0].second.timepoint +// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") +// } +// +// master?.let { volume.addChild(it) } +// +// trackFileWriter.close() +// } companion object { diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 959a32b5..02e3dbe0 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -60,6 +60,7 @@ import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import org.scijava.ui.behaviour.DragBehaviour import sc.iview.commands.demo.animation.ParticleDemo +import kotlin.properties.Delegates @Plugin(type = Command::class, menuRoot = "SciView", @@ -98,15 +99,18 @@ class VRControllerTrackingDemo: Command{ @Volatile var tracking = false var playing = false var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 + var volumesPerSecond = 1 var skipToNext = false var skipToPrevious = false // var currentVolume = 0 var volumeScaleFactor = 1.0f + var rightControllerReady = false override fun run() { + + sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -177,16 +181,17 @@ class VRControllerTrackingDemo: Command{ log.info("onDeviceConnect called, cam=${sciview.camera}") if(device.type == TrackedDeviceType.Controller) { log.info("Got device ${device.name} at $timestamp") -// if(device.role == TrackerRole.RightHand) { -// rightController = device -// log.info("rightController is found, its location is in ${rightController.position}") -// } + if(device.role == TrackerRole.RightHand) { + rightController = device + rightControllerReady = true + log.info("rightController is found and ready") + } // rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! device.model?.let { hmd.attachToNode(device, it, sciview.camera) } } } } - thread{ + thread{ inputSetup() } thread { @@ -431,31 +436,30 @@ class VRControllerTrackingDemo: Command{ volume.visible = true volume.runRecursive { it.visible = true } - playing = true + playing = false println("test") while(true) { - if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) + if(!rightControllerReady) { - println("null") + //println("null") continue } else { - rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! - + // rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! if (rightController.model?.spatialOrNull() == null) { - println("spatial null") + // println("spatial null") } else { val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() -// println(headCenter.toString()) -// println(pointWorld.toString()) + println(headCenter.toString()) + println(pointWorld.toString()) testTarget1.visible = true testTarget1.ifSpatial { position = headCenter} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index 903724b2..71f0a0bf 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -95,7 +95,7 @@ class VRHeadSetTrackingDemo: Command{ @Volatile var tracking = false var playing = false var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 + var volumesPerSecond = 1 var skipToNext = false var skipToPrevious = false // var currentVolume = 0 @@ -398,7 +398,7 @@ class VRHeadSetTrackingDemo: Command{ volume.visible = true volume.runRecursive { it.visible = true } - playing = true + playing = false while(true) @@ -434,9 +434,17 @@ class VRHeadSetTrackingDemo: Command{ spine.spatial().orientBetweenPoints(p1, p2,true,true) spine.visible = false - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + println("try to find intersection"); + val bbmin = volume.getMaximumBoundingBox().min.xyzw() + val bbmax = volume.getMaximumBoundingBox().max.xyzw() + val min = volume.spatial().world.transform(bbmin) + val max = volume.spatial().world.transform(bbmax) + println(min) + println(max) if(intersection is MaybeIntersects.Intersection) { + println("got a intersection") // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) From 9fb6c220e37c51cf50a3d18397ba8a7427133e1a Mon Sep 17 00:00:00 2001 From: ruoshan Date: Tue, 26 Jul 2022 14:36:43 +0200 Subject: [PATCH 11/55] fix issue in controller version --- build.gradle.kts | 869 ++-- settings.gradle.kts | 56 +- .../openapi/diagnostic/DefaultLogger.java | 170 +- src/main/kotlin/sc/iview/SciView.kt | 3774 ++++++++--------- .../sc/iview/commands/demo/advanced/Test.kt | 1010 ++--- .../demo/advanced/VRControllerTrackingDemo.kt | 1274 +++--- .../demo/advanced/VRHeadSetTrackingDemo.kt | 1161 ++--- 7 files changed, 4161 insertions(+), 4153 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index fd73f9dc..47207e06 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,431 +1,438 @@ -import org.gradle.kotlin.dsl.implementation -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile -import sciview.implementation -import sciview.joglNatives -import java.net.URL -import sciview.* - -plugins { - val ktVersion = "1.6.10" - val dokkaVersion = "1.6.0" - - java - kotlin("jvm") version ktVersion - kotlin("kapt") version ktVersion - sciview.publish - sciview.sign - id("org.jetbrains.dokka") version dokkaVersion - jacoco - `maven-publish` - `java-library` - signing -} - -repositories { - mavenCentral() - maven("https://maven.scijava.org/content/groups/public") - maven("https://jitpack.io") -} - -dependencies { - val ktVersion = "1.6.10" - implementation(platform("org.scijava:pom-scijava:31.1.0")) - - // Graphics dependencies - - annotationProcessor("org.scijava:scijava-common:2.87.1") - kapt("org.scijava:scijava-common:2.87.1") { // MANUAL version increment - exclude("org.lwjgl") - } - - val sceneryVersion = "71149792ec" - //val sceneryVersion = "84dfb997fc" - api("graphics.scenery:scenery:$sceneryVersion") - // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab - // if not, uncomment this only to trigger it -// api("com.github.scenerygraphics:scenery:$scenery") - - - implementation("com.fasterxml.jackson.core:jackson-databind:2.13.1") - implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.1") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.1") - implementation("org.msgpack:jackson-dataformat-msgpack:0.9.0") - - implementation(misc.cleargl) - implementation(misc.coreMem) - implementation(jogamp.jogl, joglNatives) - - implementation("com.formdev:flatlaf:1.6.5") - - // SciJava dependencies - - implementation("org.scijava:scijava-common") - implementation("org.scijava:ui-behaviour") - implementation("org.scijava:script-editor") - implementation("org.scijava:scijava-ui-swing") - implementation("org.scijava:scijava-ui-awt") - implementation("org.scijava:scijava-search") - implementation("org.scijava:scripting-jython") - implementation(migLayout.swing) - - // ImageJ dependencies - - implementation("net.imagej:imagej-common") - api("net.imagej:imagej-mesh:0.8.1") - implementation("net.imagej:imagej-mesh-io") - implementation("net.imagej:imagej-ops") - implementation("net.imagej:imagej-launcher") - implementation("net.imagej:imagej-ui-swing") - implementation("net.imagej:imagej-legacy") - implementation("io.scif:scifio") - implementation("io.scif:scifio-bf-compat") - - // ImgLib2 dependencies - implementation("net.imglib2:imglib2") - implementation("net.imglib2:imglib2-roi") - - // Math dependencies - implementation(commons.math3) - implementation(misc.joml) - - // Kotlin dependencies - implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$ktVersion") - implementation("org.jetbrains.kotlin:kotlin-stdlib:$ktVersion") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0") - - // Test scope - - testImplementation(misc.junit4) - implementation("net.imagej:ij") - implementation("net.imglib2:imglib2-ij") - - implementation(n5.core) - implementation(n5.hdf5) - implementation(n5.imglib2) - implementation("sc.fiji:spim_data") - - implementation(platform(kotlin("bom"))) - implementation(kotlin("stdlib-jdk8")) - testImplementation(kotlin("test-junit")) - - implementation("sc.fiji:bigdataviewer-core") - implementation("sc.fiji:bigdataviewer-vistools") - - // OME - implementation("ome:formats-bsd") - implementation("ome:formats-gpl") - - -} - -//kapt { -// useBuildCache = false // safe -// arguments { -// arg("-Werror") -// arg("-Xopt-in", "kotlin.RequiresOptIn") -// } -//} - -tasks { - withType().all { - val version = System.getProperty("java.version").substringBefore('.').toInt() - val default = if (version == 1) "1.8" else "$version" - kotlinOptions { - jvmTarget = project.properties["jvmTarget"]?.toString() ?: default - freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") - } - sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default - } - test { - finalizedBy(jacocoTestReport) // report is always generated after tests run - } - jar { - archiveVersion.set(rootProject.version.toString()) - } - - withType().configureEach { - val matcher = Regex("""generatePomFileFor(\w+)Publication""").matchEntire(name) - val publicationName = matcher?.let { it.groupValues[1] } - - pom.properties.empty() - - pom.withXml { - // Add parent to the generated pom - val parent = asNode().appendNode("parent") - parent.appendNode("groupId", "org.scijava") - parent.appendNode("artifactId", "pom-scijava") - parent.appendNode("version", "31.1.0") - parent.appendNode("relativePath") - - val repositories = asNode().appendNode("repositories") - val jitpackRepo = repositories.appendNode("repository") - jitpackRepo.appendNode("id", "jitpack.io") - jitpackRepo.appendNode("url", "https://jitpack.io") - - val scijavaRepo = repositories.appendNode("repository") - scijavaRepo.appendNode("id", "scijava.public") - scijavaRepo.appendNode("url", "https://maven.scijava.org/content/groups/public") - - // Update the dependencies and properties - val dependenciesNode = asNode().appendNode("dependencies") - val propertiesNode = asNode().appendNode("properties") - propertiesNode.appendNode("inceptionYear", 2016) - - // lwjgl natives - lwjglNatives.forEach { nativePlatform -> - listOf( - "", - "-glfw", - "-jemalloc", - "-opengl", - "-vulkan", - "-openvr", - "-xxhash", - "-remotery", - "-spvc", - "-shaderc", - ).forEach project@ { lwjglProject -> - // Vulkan natives only exist for macOS - if(lwjglProject.endsWith("vulkan") && nativePlatform != "macos") { - return@project - } - - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", "org.lwjgl") - dependencyNode.appendNode("artifactId", "lwjgl$lwjglProject") - dependencyNode.appendNode("version", "\${lwjgl.version}") - dependencyNode.appendNode("classifier", nativePlatform) - dependencyNode.appendNode("scope", "runtime") - } - } - - // jvrpn natives - lwjglNatives.forEach { classifier -> - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", "graphics.scenery") - dependencyNode.appendNode("artifactId", "jvrpn") - dependencyNode.appendNode("version", "\${jvrpn.version}") - dependencyNode.appendNode("classifier", classifier) - dependencyNode.appendNode("scope", "runtime") - } - // add jvrpn property because it only has runtime native deps - propertiesNode.appendNode("jvrpn.version", "1.2.0") - - val versionedArtifacts = listOf("scenery", - "flatlaf", - "kotlin-stdlib-common", - "kotlin-stdlib", - "kotlinx-coroutines-core", - "pom-scijava", - "lwjgl-bom", - "imagej-mesh", - "jackson-module-kotlin", - "jackson-dataformat-yaml", - "jackson-dataformat-msgpack", - "jogl-all", - "kotlin-bom", - "lwjgl", - "lwjgl-glfw", - "lwjgl-jemalloc", - "lwjgl-vulkan", - "lwjgl-opengl", - "lwjgl-openvr", - "lwjgl-xxhash", - "lwjgl-remotery", - "lwjgl-spvc", - "lwjgl-shaderc") - - val toSkip = listOf("pom-scijava") - - configurations.implementation.allDependencies.forEach { - val artifactId = it.name - - if (!toSkip.contains(artifactId)) { - - val propertyName = "$artifactId.version" - - if (versionedArtifacts.contains(artifactId)) { - // add "[version]" to pom - propertiesNode.appendNode(propertyName, it.version) - } - - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", it.group) - dependencyNode.appendNode("artifactId", artifactId) - dependencyNode.appendNode("version", "\${$propertyName}") - - // Custom per artifact tweaks - println(artifactId) - if ("\\-bom".toRegex().find(artifactId) != null) { - dependencyNode.appendNode("type", "pom") - } - // from https://github.com/scenerygraphics/sciview/pull/399#issuecomment-904732945 - if (artifactId == "formats-gpl") { - val exclusions = dependencyNode.appendNode("exclusions") - val jacksonCore = exclusions.appendNode("exclusion") - jacksonCore.appendNode("groupId", "com.fasterxml.jackson.core") - jacksonCore.appendNode("artifactId", "jackson-core") - val jacksonAnnotations = exclusions.appendNode("exclusion") - jacksonAnnotations.appendNode("groupId", "com.fasterxml.jackson.core") - jacksonAnnotations.appendNode("artifactId", "jackson-annotations") - } - //dependencyNode.appendNode("scope", it.scope) - } - } - - var depStartIdx = "".toRegex().find(asString())?.range?.start - var depEndIdx = "".toRegex().find(asString())?.range?.last - if (depStartIdx != null) { - if (depEndIdx != null) { - asString().replace(depStartIdx, depEndIdx + 1, "") - } - } - - depStartIdx = "".toRegex().find(asString())?.range?.start - depEndIdx = "".toRegex().find(asString())?.range?.last - if (depStartIdx != null) { - if (depEndIdx != null) { - asString().replace(depStartIdx, depEndIdx + 1, "") - } - } - } - } - - dokkaHtml { - enabled = false - dokkaSourceSets.configureEach { - sourceLink { - localDirectory.set(file("src/main/kotlin")) - remoteUrl.set(URL("https://github.com/scenerygraphics/sciview/tree/master/src/main/kotlin")) - remoteLineSuffix.set("#L") - } - } - } - - dokkaJavadoc { - enabled = false - } - - jacocoTestReport { - reports { - xml.required.set(true) - html.required.set(true) - } - dependsOn(test) // tests are required to run before generating the report - } - - register("runMain", JavaExec::class.java) { - classpath = sourceSets.main.get().runtimeClasspath - - main = "sc.iview.Main" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - - register("runImageJMain", JavaExec::class.java) { - classpath = sourceSets.main.get().runtimeClasspath - - main = "sc.iview.ImageJMain" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - - sourceSets.main.get().allSource.files - .filter { it.path.contains("demo") && (it.name.endsWith(".kt") || it.name.endsWith(".java")) } - .map { - val p = it.path - if (p.endsWith(".kt")) { - p.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - } else { - p.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - } - } - .forEach { className -> - println("Working on $className") - val exampleName = className.substringAfterLast(".") - val exampleType = className.substringBeforeLast(".").substringAfterLast(".") - - println("Registering $exampleName of $exampleType") - register(name = className.substringAfterLast(".")) { - classpath = sourceSets.test.get().runtimeClasspath - main = className - group = "demos.$exampleType" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - } - - register(name = "run") { - classpath = sourceSets.main.get().runtimeClasspath - if (project.hasProperty("target")) { - project.property("target")?.let { target -> - classpath = sourceSets.test.get().runtimeClasspath - - println("Target is $target") - // if(target.endsWith(".kt")) { - // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - // } else { - // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - // } - - main = "$target" - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - - println("Will run target $target with classpath $classpath, main=$main") - println("JVM arguments passed to target: $allJvmArgs") - } - } - } -} - -val dokkaJavadocJar by tasks.register("dokkaJavadocJar") { - dependsOn(tasks.dokkaJavadoc) - from(tasks.dokkaJavadoc.get().outputDirectory.get()) - archiveClassifier.set("javadoc") -} - -val dokkaHtmlJar by tasks.register("dokkaHtmlJar") { - dependsOn(tasks.dokkaHtml) - from(tasks.dokkaHtml.get().outputDirectory.get()) - archiveClassifier.set("html-doc") -} - -jacoco { - toolVersion = "0.8.7" -} - -artifacts { - archives(dokkaJavadocJar) - archives(dokkaHtmlJar) -} - -java.withSourcesJar() - +import org.gradle.kotlin.dsl.implementation +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import sciview.implementation +import sciview.joglNatives +import java.net.URL +import sciview.* + +plugins { + val ktVersion = "1.6.10" + val dokkaVersion = "1.6.0" + + java + kotlin("jvm") version ktVersion + kotlin("kapt") version ktVersion + sciview.publish + sciview.sign + id("org.jetbrains.dokka") version dokkaVersion + jacoco + `maven-publish` + `java-library` + signing +} + +repositories { + mavenCentral() + maven("https://maven.scijava.org/content/groups/public") + maven("https://jitpack.io") +} + +dependencies { + val ktVersion = "1.6.10" + implementation(platform("org.scijava:pom-scijava:31.1.0")) + + // Graphics dependencies + + annotationProcessor("org.scijava:scijava-common:2.87.1") + kapt("org.scijava:scijava-common:2.87.1") { // MANUAL version increment + exclude("org.lwjgl") + } + + + val sceneryVersion = "4055b8eb32" + //val sceneryVersion = "ce77dda497" + + api("graphics.scenery:scenery:$sceneryVersion") + // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab + // if not, uncomment this only to trigger it +// api("com.github.scenerygraphics:scenery:$sceneryVersion") + + api("org.apache.logging.log4j:log4j-api:2.18.0") + api("org.apache.logging.log4j:log4j-core:2.18.0") + api("org.apache.logging.log4j:log4j-1.2-api:2.18.0") + + implementation("com.fasterxml.jackson.core:jackson-databind:2.13.1") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.1") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.1") + implementation("org.msgpack:jackson-dataformat-msgpack:0.9.0") + + implementation(misc.cleargl) + implementation(misc.coreMem) + implementation(jogamp.jogl, joglNatives) + + implementation("com.formdev:flatlaf:1.6.5") + + // SciJava dependencies + + implementation("org.scijava:scijava-common") + implementation("org.scijava:ui-behaviour") + implementation("org.scijava:script-editor") + implementation("org.scijava:scijava-ui-swing") + implementation("org.scijava:scijava-ui-awt") + implementation("org.scijava:scijava-search") + implementation("org.scijava:scripting-jython") + implementation(migLayout.swing) + + // ImageJ dependencies + + implementation("net.imagej:imagej-common") + api("net.imagej:imagej-mesh:0.8.1") + implementation("net.imagej:imagej-mesh-io") + implementation("net.imagej:imagej-ops") + implementation("net.imagej:imagej-launcher") + implementation("net.imagej:imagej-ui-swing") + implementation("net.imagej:imagej-legacy") + implementation("io.scif:scifio") + implementation("io.scif:scifio-bf-compat") + + // ImgLib2 dependencies + implementation("net.imglib2:imglib2") + implementation("net.imglib2:imglib2-roi") + + // Math dependencies + implementation(commons.math3) + implementation(misc.joml) + + // Kotlin dependencies + implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$ktVersion") + implementation("org.jetbrains.kotlin:kotlin-stdlib:$ktVersion") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0") + + // Test scope + + testImplementation(misc.junit4) + implementation("net.imagej:ij") + implementation("net.imglib2:imglib2-ij") + + implementation(n5.core) + implementation(n5.hdf5) + implementation(n5.imglib2) + implementation("sc.fiji:spim_data") + + implementation(platform(kotlin("bom"))) + implementation(kotlin("stdlib-jdk8")) + testImplementation(kotlin("test-junit")) + + implementation("sc.fiji:bigdataviewer-core") + implementation("sc.fiji:bigdataviewer-vistools") + + // OME + implementation("ome:formats-bsd") + implementation("ome:formats-gpl") + + +} + +//kapt { +// useBuildCache = false // safe +// arguments { +// arg("-Werror") +// arg("-Xopt-in", "kotlin.RequiresOptIn") +// } +//} + +tasks { + withType().all { + val version = System.getProperty("java.version").substringBefore('.').toInt() + val default = if (version == 1) "1.8" else "$version" + kotlinOptions { + jvmTarget = project.properties["jvmTarget"]?.toString() ?: default + freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") + } + sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default + } + test { + finalizedBy(jacocoTestReport) // report is always generated after tests run + } + jar { + archiveVersion.set(rootProject.version.toString()) + } + + withType().configureEach { + val matcher = Regex("""generatePomFileFor(\w+)Publication""").matchEntire(name) + val publicationName = matcher?.let { it.groupValues[1] } + + pom.properties.empty() + + pom.withXml { + // Add parent to the generated pom + val parent = asNode().appendNode("parent") + parent.appendNode("groupId", "org.scijava") + parent.appendNode("artifactId", "pom-scijava") + parent.appendNode("version", "31.1.0") + parent.appendNode("relativePath") + + val repositories = asNode().appendNode("repositories") + val jitpackRepo = repositories.appendNode("repository") + jitpackRepo.appendNode("id", "jitpack.io") + jitpackRepo.appendNode("url", "https://jitpack.io") + + val scijavaRepo = repositories.appendNode("repository") + scijavaRepo.appendNode("id", "scijava.public") + scijavaRepo.appendNode("url", "https://maven.scijava.org/content/groups/public") + + // Update the dependencies and properties + val dependenciesNode = asNode().appendNode("dependencies") + val propertiesNode = asNode().appendNode("properties") + propertiesNode.appendNode("inceptionYear", 2016) + + // lwjgl natives + lwjglNatives.forEach { nativePlatform -> + listOf( + "", + "-glfw", + "-jemalloc", + "-opengl", + "-vulkan", + "-openvr", + "-xxhash", + "-remotery", + "-spvc", + "-shaderc", + ).forEach project@ { lwjglProject -> + // Vulkan natives only exist for macOS + if(lwjglProject.endsWith("vulkan") && nativePlatform != "macos") { + return@project + } + + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", "org.lwjgl") + dependencyNode.appendNode("artifactId", "lwjgl$lwjglProject") + dependencyNode.appendNode("version", "\${lwjgl.version}") + dependencyNode.appendNode("classifier", nativePlatform) + dependencyNode.appendNode("scope", "runtime") + } + } + + // jvrpn natives + lwjglNatives.forEach { classifier -> + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", "graphics.scenery") + dependencyNode.appendNode("artifactId", "jvrpn") + dependencyNode.appendNode("version", "\${jvrpn.version}") + dependencyNode.appendNode("classifier", classifier) + dependencyNode.appendNode("scope", "runtime") + } + // add lwjgl version explicitly + propertiesNode.appendNode("lwjgl.version", "3.3.1") + // add jvrpn property because it only has runtime native deps + propertiesNode.appendNode("jvrpn.version", "1.2.0") + + val versionedArtifacts = listOf("scenery", + "flatlaf", + "kotlin-stdlib-common", + "kotlin-stdlib", + "kotlinx-coroutines-core", + "pom-scijava", + "lwjgl-bom", + "imagej-mesh", + "jackson-module-kotlin", + "jackson-dataformat-yaml", + "jackson-dataformat-msgpack", + "jogl-all", + "kotlin-bom", + "lwjgl", + "lwjgl-glfw", + "lwjgl-jemalloc", + "lwjgl-vulkan", + "lwjgl-opengl", + "lwjgl-openvr", + "lwjgl-xxhash", + "lwjgl-remotery", + "lwjgl-spvc", + "lwjgl-shaderc") + + val toSkip = listOf("pom-scijava") + + configurations.implementation.allDependencies.forEach { + val artifactId = it.name + + if (!toSkip.contains(artifactId)) { + + val propertyName = "$artifactId.version" + + if (versionedArtifacts.contains(artifactId)) { + // add "[version]" to pom + propertiesNode.appendNode(propertyName, it.version) + } + + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", it.group) + dependencyNode.appendNode("artifactId", artifactId) + dependencyNode.appendNode("version", "\${$propertyName}") + + // Custom per artifact tweaks + println(artifactId) + if ("\\-bom".toRegex().find(artifactId) != null) { + dependencyNode.appendNode("type", "pom") + } + // from https://github.com/scenerygraphics/sciview/pull/399#issuecomment-904732945 + if (artifactId == "formats-gpl") { + val exclusions = dependencyNode.appendNode("exclusions") + val jacksonCore = exclusions.appendNode("exclusion") + jacksonCore.appendNode("groupId", "com.fasterxml.jackson.core") + jacksonCore.appendNode("artifactId", "jackson-core") + val jacksonAnnotations = exclusions.appendNode("exclusion") + jacksonAnnotations.appendNode("groupId", "com.fasterxml.jackson.core") + jacksonAnnotations.appendNode("artifactId", "jackson-annotations") + } + //dependencyNode.appendNode("scope", it.scope) + } + } + + var depStartIdx = "".toRegex().find(asString())?.range?.start + var depEndIdx = "".toRegex().find(asString())?.range?.last + if (depStartIdx != null) { + if (depEndIdx != null) { + asString().replace(depStartIdx, depEndIdx + 1, "") + } + } + + depStartIdx = "".toRegex().find(asString())?.range?.start + depEndIdx = "".toRegex().find(asString())?.range?.last + if (depStartIdx != null) { + if (depEndIdx != null) { + asString().replace(depStartIdx, depEndIdx + 1, "") + } + } + } + } + + dokkaHtml { + enabled = false + dokkaSourceSets.configureEach { + sourceLink { + localDirectory.set(file("src/main/kotlin")) + remoteUrl.set(URL("https://github.com/scenerygraphics/sciview/tree/master/src/main/kotlin")) + remoteLineSuffix.set("#L") + } + } + } + + dokkaJavadoc { + enabled = false + } + + jacocoTestReport { + reports { + xml.required.set(true) + html.required.set(true) + } + dependsOn(test) // tests are required to run before generating the report + } + + register("runMain", JavaExec::class.java) { + classpath = sourceSets.main.get().runtimeClasspath + + main = "sc.iview.Main" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + + register("runImageJMain", JavaExec::class.java) { + classpath = sourceSets.main.get().runtimeClasspath + + main = "sc.iview.ImageJMain" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + + sourceSets.main.get().allSource.files + .filter { it.path.contains("demo") && (it.name.endsWith(".kt") || it.name.endsWith(".java")) } + .map { + val p = it.path + if (p.endsWith(".kt")) { + p.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + } else { + p.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + } + } + .forEach { className -> + println("Working on $className") + val exampleName = className.substringAfterLast(".") + val exampleType = className.substringBeforeLast(".").substringAfterLast(".") + + println("Registering $exampleName of $exampleType") + register(name = className.substringAfterLast(".")) { + classpath = sourceSets.test.get().runtimeClasspath + main = className + group = "demos.$exampleType" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + } + + register(name = "run") { + classpath = sourceSets.main.get().runtimeClasspath + if (project.hasProperty("target")) { + project.property("target")?.let { target -> + classpath = sourceSets.test.get().runtimeClasspath + + println("Target is $target") + // if(target.endsWith(".kt")) { + // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + // } else { + // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + // } + + main = "$target" + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + + println("Will run target $target with classpath $classpath, main=$main") + println("JVM arguments passed to target: $allJvmArgs") + } + } + } +} + +val dokkaJavadocJar by tasks.register("dokkaJavadocJar") { + dependsOn(tasks.dokkaJavadoc) + from(tasks.dokkaJavadoc.get().outputDirectory.get()) + archiveClassifier.set("javadoc") +} + +val dokkaHtmlJar by tasks.register("dokkaHtmlJar") { + dependsOn(tasks.dokkaHtml) + from(tasks.dokkaHtml.get().outputDirectory.get()) + archiveClassifier.set("html-doc") +} + +jacoco { + toolVersion = "0.8.7" +} + +artifacts { + archives(dokkaJavadocJar) + archives(dokkaHtmlJar) +} + +java.withSourcesJar() + diff --git a/settings.gradle.kts b/settings.gradle.kts index afe04a9f..cc446b56 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,28 +1,28 @@ - -pluginManagement { - repositories { - gradlePluginPortal() - maven("https://raw.githubusercontent.com/kotlin-graphics/mary/master") - } -} - -plugins { - id("sciJava.catalogs") version "30.0.0+66" -} - -rootProject.name = "sciview" - -gradle.rootProject { - group = "graphics.scenery" - version = "0.2.0-beta-9-SNAPSHOT-test-4" - description = "Scenery-backed 3D visualization package for ImageJ." -} - -val useLocalScenery: String? by extra -if (System.getProperty("CI").toBoolean() != true - && System.getenv("CI").toBoolean() != true - && useLocalScenery?.toBoolean() == true) - if(File("../scenery/build.gradle.kts").exists()) { - logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") - includeBuild("../scenery") - } + +pluginManagement { + repositories { + gradlePluginPortal() + maven("https://raw.githubusercontent.com/kotlin-graphics/mary/master") + } +} + +plugins { + id("sciJava.catalogs") version "30.0.0+66" +} + +rootProject.name = "sciview" + +gradle.rootProject { + group = "sc.iview" + version = "0.2.0-beta-9-SNAPSHOT-test-6" + description = "Scenery-backed 3D visualization package for ImageJ." +} + +val useLocalScenery: String? by extra +if (System.getProperty("CI").toBoolean() != true + && System.getenv("CI").toBoolean() != true + && useLocalScenery?.toBoolean() == true) + if(File("../scenery/build.gradle.kts").exists()) { + logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") + includeBuild("../scenery") + } diff --git a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java index 91c12ad8..2af3c655 100644 --- a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java +++ b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java @@ -1,84 +1,86 @@ -/* - * Copyright 2000-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.openapi.diagnostic; - -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.util.ExceptionUtil; -import org.apache.log4j.Level; -import org.jetbrains.annotations.NonNls; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -public class DefaultLogger extends Logger { - @SuppressWarnings("UnusedParameters") - public DefaultLogger(String category) { } - - @Override - public boolean isDebugEnabled() { - return false; - } - - @Override - public void debug(String message) { } - - @Override - public void debug(Throwable t) { } - - @Override - public void debug(@NonNls String message, Throwable t) { } - - @Override - public void info(String message) { } - - @Override - public void info(String message, Throwable t) { } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void warn(@NonNls String message, @Nullable Throwable t) { - t = checkException(t); - System.err.println("WARN: " + message); - if (t != null) t.printStackTrace(System.err); - } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void error(String message, @Nullable Throwable t, @NotNull String... details) { - t = checkException(t); - message += attachmentsToString(t); - System.err.println("ERROR: " + message); - if (t != null) t.printStackTrace(System.err); - if (details.length > 0) { - System.out.println("details: "); - for (String detail : details) { - System.out.println(detail); - } - } - - AssertionError error = new AssertionError(message); - error.initCause(t); - throw error; - } - - @Override - public void setLevel(Level level) { } - - public static String attachmentsToString(@Nullable Throwable t) { - //noinspection ThrowableResultOfMethodCallIgnored - Throwable rootCause = t == null ? null : ExceptionUtil.getRootCause(t); - return ""; - } -} +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.openapi.diagnostic; + +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.ExceptionUtil; +import org.apache.log4j.Level; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + + +public class DefaultLogger extends Logger { + @SuppressWarnings("UnusedParameters") + public DefaultLogger(String category) { } + + @Override + public boolean isDebugEnabled() { + return false; + } + + @Override + public void debug(String message) { } + + @Override + public void debug(Throwable t) { } + + @Override + public void debug(@NonNls String message, Throwable t) { } + + @Override + public void info(String message) { } + + @Override + public void info(String message, Throwable t) { } + + @Override + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void warn(@NonNls String message, @Nullable Throwable t) { + t = checkException(t); + System.err.println("WARN: " + message); + if (t != null) t.printStackTrace(System.err); + } + + @Override + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void error(String message, @Nullable Throwable t, @NotNull String... details) { + t = checkException(t); + message += attachmentsToString(t); + System.err.println("ERROR: " + message); + if (t != null) t.printStackTrace(System.err); + if (details.length > 0) { + System.out.println("details: "); + for (String detail : details) { + System.out.println(detail); + } + } + + AssertionError error = new AssertionError(message); + error.initCause(t); + throw error; + } + + @Override + public void setLevel(Level level) { } + + public static String attachmentsToString(@Nullable Throwable t) { + //noinspection ThrowableResultOfMethodCallIgnored + Throwable rootCause = t == null ? null : ExceptionUtil.getRootCause(t); + return ""; + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 4a0fef21..5c15fa5e 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -1,1887 +1,1887 @@ -/*- - * #%L - * Scenery-backed 3D visualization package for ImageJ. - * %% - * Copyright (C) 2016 - 2021 SciView developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package sc.iview - -import bdv.BigDataViewer -import bdv.cache.CacheControl -import bdv.tools.brightness.ConverterSetup -import bdv.util.AxisOrder -import bdv.util.RandomAccessibleIntervalSource -import bdv.util.RandomAccessibleIntervalSource4D -import bdv.util.volatiles.VolatileView -import bdv.viewer.Source -import bdv.viewer.SourceAndConverter -import graphics.scenery.* -import graphics.scenery.Scene.RaycastResult -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.opengl.OpenGLRenderer -import graphics.scenery.backends.vulkan.VulkanRenderer -import graphics.scenery.controls.InputHandler -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerInput -import graphics.scenery.primitives.* -import graphics.scenery.proteins.Protein -import graphics.scenery.proteins.RibbonDiagram -import graphics.scenery.utils.ExtractsNatives -import graphics.scenery.utils.ExtractsNatives.Companion.getPlatform -import graphics.scenery.utils.LogbackUtils -import graphics.scenery.utils.SceneryPanel -import graphics.scenery.utils.Statistics -import graphics.scenery.utils.extensions.times -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.RAIVolume -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import graphics.scenery.volumes.Volume.Companion.fromXML -import graphics.scenery.volumes.Volume.Companion.setupId -import graphics.scenery.volumes.Volume.VolumeDataSource.RAISource -import io.scif.SCIFIOService -import io.scif.services.DatasetIOService -import net.imagej.Dataset -import net.imagej.ImageJService -import net.imagej.axis.CalibratedAxis -import net.imagej.axis.DefaultAxisType -import net.imagej.axis.DefaultLinearAxis -import net.imagej.interval.CalibratedRealInterval -import net.imagej.lut.LUTService -import net.imagej.mesh.Mesh -import net.imagej.mesh.io.ply.PLYMeshIO -import net.imagej.mesh.io.stl.STLMeshIO -import net.imagej.units.UnitService -import net.imglib2.* -import net.imglib2.display.ColorTable -import net.imglib2.img.Img -import net.imglib2.img.array.ArrayImgs -import net.imglib2.realtransform.AffineTransform3D -import net.imglib2.type.numeric.ARGBType -import net.imglib2.type.numeric.NumericType -import net.imglib2.type.numeric.RealType -import net.imglib2.type.numeric.integer.UnsignedByteType -import net.imglib2.view.Views -import org.joml.Quaternionf -import org.joml.Vector3f -import org.joml.Vector4f -import org.scijava.Context -import org.scijava.`object`.ObjectService -import org.scijava.display.Display -import org.scijava.event.EventHandler -import org.scijava.event.EventService -import org.scijava.io.IOService -import org.scijava.log.LogLevel -import org.scijava.log.LogService -import org.scijava.menu.MenuService -import org.scijava.plugin.Parameter -import org.scijava.service.SciJavaService -import org.scijava.thread.ThreadService -import org.scijava.util.ColorRGB -import org.scijava.util.Colors -import org.scijava.util.VersionUtils -import sc.iview.commands.demo.animation.ParticleDemo -import sc.iview.event.NodeActivatedEvent -import sc.iview.event.NodeAddedEvent -import sc.iview.event.NodeChangedEvent -import sc.iview.event.NodeRemovedEvent -import sc.iview.process.MeshConverter -import sc.iview.ui.CustomPropertyUI -import sc.iview.ui.MainWindow -import sc.iview.ui.SwingMainWindow -import sc.iview.ui.TaskManager -import tpietzsch.example2.VolumeViewerOptions -import java.awt.event.WindowListener -import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.FloatBuffer -import java.nio.file.Path -import java.util.* -import java.util.concurrent.Future -import java.util.function.Consumer -import java.util.function.Function -import java.util.function.Predicate -import java.util.stream.Collectors -import kotlin.collections.ArrayList -import kotlin.collections.HashMap -import kotlin.math.cos -import kotlin.math.sin -import kotlin.system.measureTimeMillis - -/** - * Main SciView class. - * - * @author Kyle Harrington - */ -// we suppress unused warnings here because @Parameter-annotated fields -// get updated automatically by SciJava. -class SciView : SceneryBase, CalibratedRealInterval { - val sceneryPanel = arrayOf(null) - - /* - * Return the default floor object - *//* - * Set the default floor object - */ - /** - * The floor that orients the user in the scene - */ - var floor: Node? = null - protected var vrActive = false - - /** - * The primary camera/observer in the scene - */ - var camera: Camera? = null - set(value) { - field = value - setActiveObserver(field) - } - - lateinit var controls: Controls - val targetArcball: AnimatedCenteringBeforeArcBallControl - get() = controls.targetArcball - - val currentScene: Scene - get() = scene - - /** - * Geometry/Image information of scene - */ - private lateinit var axes: Array - - @Parameter - private lateinit var log: LogService - - @Parameter - private lateinit var menus: MenuService - - @Parameter - private lateinit var io: IOService - - @Parameter - private lateinit var eventService: EventService - - @Parameter - private lateinit var lutService: LUTService - - @Parameter - private lateinit var threadService: ThreadService - - @Parameter - private lateinit var objectService: ObjectService - - @Parameter - private lateinit var unitService: UnitService - - private lateinit var imageToVolumeMap: HashMap - - /** - * Queue keeps track of the currently running animations - */ - private var animations: Queue>? = null - - /** - * Animation pause tracking - */ - private var animating = false - - /** - * This tracks the actively selected Node in the scene - */ - var activeNode: Node? = null - private set - - /* - * Return the SciJava Display that contains SciView - *//* - * Set the SciJava Display - */ var display: Display<*>? = null - - /** - * Return the current SceneryJPanel. This is necessary for custom context menus - * @return panel the current SceneryJPanel - */ - var lights: ArrayList? = null - private set - private val notAbstractNode: Predicate = Predicate { node: Node -> !(node is Camera || node is Light || node === floor) } - var isClosed = false - internal set - - private val notAbstractBranchingFunction = Function { node: Node -> node.children.stream().filter(notAbstractNode).collect(Collectors.toList()) } - - val taskManager = TaskManager() - - // If true, then when a new node is added to the scene, the camera will refocus on this node by default - var centerOnNewNodes = false - - // If true, then when a new node is added the thread will block until the node is added to the scene. This is required for - // centerOnNewNodes - var blockOnNewNodes = false - private var headlight: PointLight? = null - - lateinit var mainWindow: MainWindow - - constructor(context: Context) : super("SciView", 1280, 720, false, context) { - context.inject(this) - } - - constructor(applicationName: String?, windowWidth: Int, windowHeight: Int) : super(applicationName!!, windowWidth, windowHeight, false) - - fun publicGetInputHandler(): InputHandler { - return inputHandler!! - } - - /** - * Toggle video recording with scenery's video recording mechanism - * Note: this video recording may skip frames because it is asynchronous - */ - fun toggleRecordVideo() { - if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie() else (renderer as VulkanRenderer).recordMovie() - } - - /** - * Toggle video recording with scenery's video recording mechanism - * Note: this video recording may skip frames because it is asynchronous - * - * @param filename destination for saving video - * @param overwrite should the file be replaced, otherwise a unique incrementing counter will be appended - */ - fun toggleRecordVideo(filename: String?, overwrite: Boolean) { - if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie(filename!!, overwrite) else (renderer as VulkanRenderer).recordMovie(filename!!, overwrite) - } - - /** - * See [Controls.stashControls]. - */ - fun stashControls() { - controls.stashControls() - } - - /** - * See [Controls.restoreControls] and [Controls.stashControls]. - */ - fun restoreControls() { - controls.restoreControls() - } - - internal fun setRenderer(newRenderer: Renderer) { - renderer = newRenderer - } - - - /** - * Reset the scene to initial conditions - */ - fun reset() { - // Initialize the 3D axes - axes = arrayOf( - DefaultLinearAxis(DefaultAxisType("X", true), "um", 1.0), - DefaultLinearAxis(DefaultAxisType("Y", true), "um", 1.0), - DefaultLinearAxis(DefaultAxisType("Z", true), "um", 1.0) - ) - - // Remove everything except camera - val toRemove = getSceneNodes { n: Node? -> n !is Camera } - for (n in toRemove) { - deleteNode(n, false) - } - - imageToVolumeMap = HashMap() - - // Setup camera - if (camera == null) { - camera = DetachedHeadCamera() - (camera as DetachedHeadCamera).position = Vector3f(0.0f, 1.65f, 0.0f) - scene.addChild(camera as DetachedHeadCamera) - } - camera!!.spatial().position = Vector3f(0.0f, 1.65f, 5.0f) - camera!!.perspectiveCamera(50.0f, windowWidth, windowHeight, 0.1f, 1000.0f) - - // Setup lights - val tetrahedron = arrayOfNulls(4) - tetrahedron[0] = Vector3f(1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[1] = Vector3f(-1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[2] = Vector3f(0.0f, 1.0f, 1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[3] = Vector3f(0.0f, -1.0f, 1.0f / Math.sqrt(2.0).toFloat()) - lights = ArrayList() - for (i in 0..3) { // TODO allow # initial lights to be customizable? - val light = PointLight(150.0f) - light.spatial().position = tetrahedron[i]!!.mul(25.0f) - light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) - light.intensity = 1.0f - lights!!.add(light) - //camera.addChild( light ); - scene.addChild(light) - } - - // Make a headlight for the camera - headlight = PointLight(150.0f) - headlight!!.spatial().position = Vector3f(0f, 0f, -1f).mul(25.0f) - headlight!!.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) - headlight!!.intensity = 0.5f - headlight!!.name = "headlight" - val lightSphere = Icosphere(1.0f, 2) - headlight!!.addChild(lightSphere) - lightSphere.material().diffuse = headlight!!.emissionColor - lightSphere.material().specular = headlight!!.emissionColor - lightSphere.material().ambient = headlight!!.emissionColor - lightSphere.material().wireframe = true - lightSphere.visible = false - //lights.add( light ); - camera!!.nearPlaneDistance = 0.01f - camera!!.farPlaneDistance = 1000.0f - camera!!.addChild(headlight!!) - floor = InfinitePlane() //new Box( new Vector3f( 500f, 0.2f, 500f ) ); - (floor as InfinitePlane).type = InfinitePlane.Type.Grid - (floor as Node).name = "Floor" - scene.addChild(floor as Node) - } - - /** - * Initialization of SWING and scenery. Also triggers an initial population of lights/camera in the scene - */ - override fun init() { - val logLevel = System.getProperty("scenery.LogLevel", "info") - log.level = LogLevel.value(logLevel) - LogbackUtils.setLogLevel(null, logLevel) - System.getProperties().stringPropertyNames().forEach(Consumer { name: String -> - if (name.startsWith("scenery.LogLevel")) { - LogbackUtils.setLogLevel("", System.getProperty(name, "info")) - } - }) - - // determine imagej-launcher version and to disable Vulkan if XInitThreads() fix - // is not deployed - try { - val launcherClass = Class.forName("net.imagej.launcher.ClassLauncher") - var versionString = VersionUtils.getVersion(launcherClass) - if (versionString != null && getPlatform() == ExtractsNatives.Platform.LINUX) { - versionString = versionString.substring(0, 5) - val launcherVersion = Version(versionString) - val nonWorkingVersion = Version("4.0.5") - if (launcherVersion.compareTo(nonWorkingVersion) <= 0 - && !java.lang.Boolean.parseBoolean(System.getProperty("sciview.DisableLauncherVersionCheck", "false"))) { - logger.info("imagej-launcher version smaller or equal to non-working version ($versionString vs. 4.0.5), disabling Vulkan as rendering backend. Disable check by setting 'scenery.DisableLauncherVersionCheck' system property to 'true'.") - System.setProperty("scenery.Renderer", "OpenGLRenderer") - } else { - logger.info("imagej-launcher version bigger that non-working version ($versionString vs. 4.0.5), all good.") - } - } - } catch (cnfe: ClassNotFoundException) { - // Didn't find the launcher, so we're probably good. - logger.info("imagej-launcher not found, not touching renderer preferences.") - } - - animations = LinkedList() - mainWindow = SwingMainWindow(this) - controls = Controls(this) - - imageToVolumeMap = HashMap() - } - - fun toggleSidebar(): Boolean { - return mainWindow.toggleSidebar() - - } - - private fun initializeInterpreter() { - mainWindow.initializeInterpreter() - } - - /* - * Completely close the SciView window + cleanup - */ - fun closeWindow() { - mainWindow.close() - dispose() - } - - /* - * Return true if the scene has been initialized - */ - val isInitialized: Boolean - get() = sceneInitialized() - - /** - * Place the scene into the center of camera view, and zoom in/out such - * that the whole scene is in the view (everything would be visible if it - * would not be potentially occluded). - */ - fun fitCameraToScene() { - centerOnNode(scene) - //TODO: smooth zoom in/out VLADO vlado Vlado - } - - /** - * Place the scene into the center of camera view. - */ - fun centerOnScene() { - centerOnNode(scene) - } - /* - * Get the InputHandler that is managing mouse, input, VR controls, etc. - */ - val sceneryInputHandler: InputHandler - get() = inputHandler!! - - /* - * Return a bounding box around a subgraph of the scenegraph - */ - fun getSubgraphBoundingBox(n: Node): OrientedBoundingBox? { - val predicate = Function> { node: Node -> node.children } - return getSubgraphBoundingBox(n, predicate) - } - - /* - * Return a bounding box around a subgraph of the scenegraph - */ - fun getSubgraphBoundingBox(n: Node, branchFunction: Function>): OrientedBoundingBox? { - if (n.boundingBox == null && n.children.size != 0) { - return n.getMaximumBoundingBox().asWorld() - } - val branches = branchFunction.apply(n) - if (branches.isEmpty()) { - return if (n.boundingBox == null) null else n.boundingBox!!.asWorld() - } - var bb = n.getMaximumBoundingBox() - for (c in branches) { - val cBB = getSubgraphBoundingBox(c, branchFunction) - if (cBB != null) bb = bb.expand(bb, cBB) - } - return bb - } - - /** - * Place the active node into the center of camera view. - */ - fun centerOnActiveNode() { - if (activeNode == null) return - centerOnNode(activeNode) - } - - /** - * Place the specified node into the center of camera view. - */ - fun centerOnNode(currentNode: Node?) { - if (currentNode == null) { - log.info("Cannot center on null node.") - return - } - - //center the on the same spot as ArcBall does - centerOnPosition(currentNode.getMaximumBoundingBox().getBoundingSphere().origin) - } - - /** - * Center the camera on the specified Node - */ - fun centerOnPosition(currentPos: Vector3f?) { - controls.centerOnPosition(currentPos) - } - - /** - * Activate the node, and center the view on it. - * @param n - * @return the currently active node - */ - fun setActiveCenteredNode(n: Node?): Node? { - //activate... - val ret = setActiveNode(n) - //...and center it - ret?.let { centerOnNode(it) } - return ret - } - - //a couple of shortcut methods to readout controls params - fun getFPSSpeedSlow(): Float { - return controls.getFPSSpeedSlow() - } - - fun getFPSSpeedFast(): Float { - return controls.getFPSSpeedFast() - } - - fun getFPSSpeedVeryFast(): Float { - return controls.getFPSSpeedVeryFast() - } - - fun getMouseSpeed(): Float { - return controls.getMouseSpeed() - } - - fun getMouseScrollSpeed(): Float { - return controls.getMouseScrollSpeed() - } - - //a couple of setters with scene sensible boundary checks - fun setFPSSpeedSlow(slowSpeed: Float) { - controls.setFPSSpeedSlow(slowSpeed) - } - - fun setFPSSpeedFast(fastSpeed: Float) { - controls.setFPSSpeedFast(fastSpeed) - } - - fun setFPSSpeedVeryFast(veryFastSpeed: Float) { - controls.setFPSSpeedVeryFast(veryFastSpeed) - } - - fun setFPSSpeed(newBaseSpeed: Float) { - controls.setFPSSpeed(newBaseSpeed) - } - - fun setMouseSpeed(newSpeed: Float) { - controls.setMouseSpeed(newSpeed) - } - - fun setMouseScrollSpeed(newSpeed: Float) { - controls.setMouseScrollSpeed(newSpeed) - } - - fun setObjectSelectionMode() { - controls.setObjectSelectionMode() - } - - /* - * Set the action used during object selection - */ - fun setObjectSelectionMode(selectAction: Function3?) { - controls.setObjectSelectionMode(selectAction) - } - - fun showContextNodeChooser(x: Int, y: Int) { - mainWindow.showContextNodeChooser(x,y) - } - - /* - * Initial configuration of the scenery InputHandler - * This is automatically called and should not be used directly - */ - override fun inputSetup() { - log.info("Running InputSetup") - controls.inputSetup() - } - - /** - * Add a box at the specified position with specified size, color, and normals on the inside/outside - * @param position position to put the box - * @param size size of the box - * @param color color of the box - * @param inside are normals inside the box? - * @return the Node corresponding to the box - */ - @JvmOverloads - fun addBox(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), size: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, - inside: Boolean = false, block: Box.() -> Unit = {}): Box { - val box = Box(size, inside) - box.spatial().position = position - box.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - return addNode(box, block = block) - } - - /** - * Add a unit sphere at a given [position] with given [radius] and [color]. - * @return the Node corresponding to the sphere - */ - @JvmOverloads - fun addSphere(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), radius: Float = 1f, color: ColorRGB = DEFAULT_COLOR, block: Sphere.() -> Unit = {}): Sphere { - val sphere = Sphere(radius, 20) - sphere.spatial().position = position - sphere.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - - return addNode(sphere, block = block) - } - - /** - * Add a Cylinder at the given position with radius, height, and number of faces/segments - * @param position position of the cylinder - * @param radius radius of the cylinder - * @param height height of the cylinder - * @param num_segments number of segments to represent the cylinder - * @return the Node corresponding to the cylinder - */ - fun addCylinder(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cylinder.() -> Unit = {}): Cylinder { - val cyl = Cylinder(radius, height, num_segments) - cyl.spatial().position = position - return addNode(cyl, block = block) - } - - /** - * Add a Cone at the given position with radius, height, and number of faces/segments - * @param position position to put the cone - * @param radius radius of the cone - * @param height height of the cone - * @param num_segments number of segments used to represent cone - * @return the Node corresponding to the cone - */ - fun addCone(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cone.() -> Unit = {}): Cone { - val cone = Cone(radius, height, num_segments, Vector3f(0.0f, 0.0f, 1.0f)) - cone.spatial().position = position - return addNode(cone, block = block) - } - - /** - * Add a line from start to stop with the given color - * @param start start position of line - * @param stop stop position of line - * @param color color of line - * @return the Node corresponding to the line - */ - @JvmOverloads - fun addLine(start: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), stop: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, block: Line.() -> Unit = {}): Line { - return addLine(arrayOf(start, stop), color, 0.1, block) - } - - /** - * Add a multi-segment line that goes through the supplied points with a single color and edge width - * @param points points along line including first and terminal points - * @param color color of line - * @param edgeWidth width of line segments - * @return the Node corresponding to the line - */ - @JvmOverloads - fun addLine(points: Array, color: ColorRGB, edgeWidth: Double, block: Line.() -> Unit = {}): Line { - val line = Line(points.size) - for (pt in points) { - line.addPoint(pt) - } - line.edgeWidth = edgeWidth.toFloat() - line.material { - ambient = Vector3f(1.0f, 1.0f, 1.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - line.spatial().position = points[0] - return addNode(line, block = block) - } - - /** - * Add a PointLight source at the origin - * @return a Node corresponding to the PointLight - */ - @JvmOverloads - fun addPointLight(block: PointLight.() -> Unit = {}): PointLight { - val light = PointLight(5.0f) - light.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Vector3f(0.0f, 1.0f, 0.0f) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - light.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - lights!!.add(light) - return addNode(light, block = block) - } - - /** - * Position all lights that were initialized by default around the scene in a circle at Y=0 - */ - fun surroundLighting() { - val bb = getSubgraphBoundingBox(scene, notAbstractBranchingFunction) - val (c, r) = bb!!.getBoundingSphere() - // Choose a good y-position, then place lights around the cross-section through this plane - val y = 0f - for (k in lights!!.indices) { - val light = lights!![k] - val x = (c.x() + r * cos(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() - val z = (c.y() + r * sin(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() - light.lightRadius = 2 * r - light.spatial().position = Vector3f(x, y, z) - } - } - - @Throws(IOException::class) - fun openDirTiff(source: Path) - { - val v = Volume.fromPath(source, hub) - v.name = "volume" - v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) - v.colormap = Colormap.get("jet") - v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) - v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) - v.metadata["animating"] = true - v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) - v.visible = true - - v.spatial().wantsComposeModel = true - v.spatial().updateWorld(true) - System.out.println("v.model: " + v.model) - addChild(v) - System.out.println("v.getDimensions: "+ v.getDimensions()) - - System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) - System.out.println("v.world.matrix: " + v.spatial().world) - } - - data class PointInTrack( - val t: Int, - val loc: Vector3f, - val cellId: Long, - val parentId: Long, - val nodeScore: Float, - val edgeScore: Float - ) - - data class Track( - val track: List, - val trackId: Int - ) - - @Throws(IOException::class) - fun openTrackFile(file: File) - { - val lines = file.readLines() - var track = ArrayList() - val tracks = ArrayList() - val separator = "," - - var lastTrackId = -1 - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val t = tokens[0].toInt() - val z = tokens[1].toFloat() -2000f - val y = tokens[2].toFloat() -800f - val x = tokens[3].toFloat() -1300f - val cellId = tokens[4].toLong() - val parentId = tokens[5].toLong() - val trackId = tokens[6].toInt() - val nodeScore = tokens[7].toFloat() - val edgeScore = tokens[8].toFloat()/45.0f - - val currentPointInTrack = PointInTrack( - t, - Vector3f(x,y,z), - cellId, - parentId, - nodeScore, - edgeScore - ) - if(lastTrackId != trackId) - { - lastTrackId = trackId - val sortedTrack = track.sortedBy { it.t } - tracks.add(Track(sortedTrack, trackId)) - - track.clear() - } - track.add(currentPointInTrack) - } - val timeCost = measureTimeMillis { - addTracks(tracks) - } - println("time: $timeCost") - } - - fun addTracks(tracks: ArrayList) - { - val rng = Random(17) - for(track in tracks) - { - if(track.trackId > 10) - { - continue - } - System.out.println("add track: "+ track.trackId.toString() ) - val master = Cylinder(0.1f, 1.0f, 10) -// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Vector3f(0.05f, 0f, 0f) - metallic = 0.01f - roughness = 0.5f - } - - val mInstanced = InstancedNode(master) - mInstanced.name = "TrackID-${track.trackId}" - mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } - addNode(mInstanced) - - var cnt = 0 - val a = rng.nextFloat() - val b = rng.nextFloat() - track.track.windowed(2,1).forEach { pair -> - cnt = cnt + 1 - val element = mInstanced.addInstance() - element.name ="EdgeID-$cnt" - element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} - element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) - //mInstanced.instances.add(element) - - } - - } - - } - - - /** - * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud - * @param source string of a data source - * @throws IOException - */ - @Suppress("UNCHECKED_CAST") - @Throws(IOException::class) - fun open(source: String) { - if (source.endsWith(".xml", ignoreCase = true)) { - addNode(fromXML(source, hub, VolumeViewerOptions())) - return - } else if (source.takeLast(4).equals(".pdb", true)) { - val protein = Protein.fromFile(source) - val ribbon = RibbonDiagram(protein) - ribbon.spatial().position = Vector3f(0f, 0f, 0f) - addNode(ribbon) - return - } else if (source.endsWith(".stl", ignoreCase = true)) { - val stlReader = STLMeshIO() - addMesh(stlReader.open(source)) - return - } else if (source.endsWith(".ply", ignoreCase = true)) { - val plyReader = PLYMeshIO() - addMesh(plyReader.open(source)) - return - } - val data = io.open(source) - if (data is Mesh) - addMesh(data) - else if (data is graphics.scenery.Mesh) - addMesh(data) - else if (data is PointCloud) - addPointCloud(data) - else if (data is Dataset) - addVolume(data) - else if (data is RandomAccessibleInterval<*>) - addVolume(data as RandomAccessibleInterval>, source) - else if (data is List<*>) { - val list = data - require(!list.isEmpty()) { "Data source '$source' appears empty." } - val element = list[0] - if (element is RealLocalizable) { - // NB: For now, we assume all elements will be RealLocalizable. - // Highly likely to be the case, barring antagonistic importers. - val points = list as List - addPointCloud(points, source) - } else { - val type = if (element == null) "" else element.javaClass.name - throw IllegalArgumentException("Data source '" + source + // - "' contains elements of unknown type '" + type + "'") - } - } else { - val type = if (data == null) "" else data.javaClass.name - throw IllegalArgumentException("Data source '" + source + // - "' contains data of unknown type '" + type + "'") - } - } - - /** - * Add the given points to the scene as a PointCloud with a given name - * @param points points to use in a PointCloud - * @param name name of the PointCloud - * @return - */ - @JvmOverloads - fun addPointCloud(points: Collection, - name: String? = "PointCloud", - pointSize : Float = 1.0f, - block: PointCloud.() -> Unit = {}): PointCloud { - val flatVerts = FloatArray(points.size * 3) - var k = 0 - for (point in points) { - flatVerts[k * 3] = point.getFloatPosition(0) - flatVerts[k * 3 + 1] = point.getFloatPosition(1) - flatVerts[k * 3 + 2] = point.getFloatPosition(2) - k++ - } - val pointCloud = PointCloud(pointSize, name!!) - val vBuffer: FloatBuffer = BufferUtils.allocateFloat(flatVerts.size * 4) - val nBuffer: FloatBuffer = BufferUtils.allocateFloat(0) - vBuffer.put(flatVerts) - vBuffer.flip() - pointCloud.geometry().vertices = vBuffer - pointCloud.geometry().normals = nBuffer - pointCloud.geometry().indices = BufferUtils.allocateInt(0) - pointCloud.spatial().position = Vector3f(0f, 0f, 0f) - - pointCloud.setupPointCloud() - return addNode(pointCloud, block = block) - } - - /** - * Add a PointCloud to the scene - * @param pointCloud existing PointCloud to add to scene - * @return a Node corresponding to the PointCloud - */ - @JvmOverloads - fun addPointCloud(pointCloud: PointCloud, block: PointCloud.() -> Unit = {}): PointCloud { - pointCloud.setupPointCloud() - pointCloud.spatial().position = Vector3f(0f, 0f, 0f) - return addNode(pointCloud, block = block) - } - - /** - * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true - * @param n node to add to scene - * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services - * @return a Node corresponding to the Node - */ - @JvmOverloads - fun addNode(n: N, activePublish: Boolean = true, block: N.() -> Unit = {}): N { - n?.let { - it.block() - scene.addChild(it) - objectService.addObject(n) - if (blockOnNewNodes) { - Utils.blockWhile({ this.find(n.name) == null }, 20) - //System.out.println("find(name) " + find(n.getName()) ); - } - // Set new node as active and centered? - setActiveNode(n) - if (centerOnNewNodes) { - centerOnNode(n) - } - if (activePublish) { - eventService.publish(NodeAddedEvent(n)) - } - } - return n - } - - /** - * Add a scenery Mesh to the scene - * @param scMesh scenery mesh to add to scene - * @return a Node corresponding to the mesh - */ - fun addMesh(scMesh: graphics.scenery.Mesh): graphics.scenery.Mesh { - scMesh.ifMaterial { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Vector3f(0.0f, 1.0f, 0.0f) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - scMesh.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - objectService.addObject(scMesh) - return addNode(scMesh) - } - - /** - * Add an ImageJ mesh to the scene - * @param mesh net.imagej.mesh to add to scene - * @return a Node corresponding to the mesh - */ - fun addMesh(mesh: Mesh): graphics.scenery.Mesh { - val scMesh = MeshConverter.toScenery(mesh) - return addMesh(scMesh) - } - - /** - * [Deprecated: use deleteNode] - * Remove a Mesh from the scene - * @param scMesh mesh to remove from scene - */ - fun removeMesh(scMesh: graphics.scenery.Mesh?) { - scene.removeChild(scMesh!!) - } - - /** - * Set the currently active node - * @param n existing node that should become active focus of this SciView - * @return the currently active node - */ - fun setActiveNode(n: Node?): Node? { - if (activeNode === n) return activeNode - activeNode = n - targetArcball.target = { n?.getMaximumBoundingBox()?.getBoundingSphere()?.origin ?: Vector3f(0.0f, 0.0f, 0.0f) } - mainWindow.selectNode(activeNode) - eventService.publish(NodeActivatedEvent(activeNode)) - return activeNode - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeAdded(event: NodeAddedEvent?) { - mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeRemoved(event: NodeRemovedEvent?) { - mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeChanged(event: NodeChangedEvent?) { - // TODO: Check if rebuilding the tree is necessary here, otherwise this costs a lot of performance - //mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeActivated(event: NodeActivatedEvent?) { - // TODO: add listener code for node activation, if necessary - // NOTE: do not update property window here, this will lead to a loop. - } - - fun toggleInspectorWindow() { - toggleSidebar() - } - - @Suppress("UNUSED_PARAMETER") - fun setInspectorWindowVisibility(visible: Boolean) { -// inspector.setVisible(visible); -// if( visible ) -// mainSplitPane.setDividerLocation(getWindowWidth()/4 * 3); -// else -// mainSplitPane.setDividerLocation(getWindowWidth()); - } - - @Suppress("UNUSED_PARAMETER") - fun setInterpreterWindowVisibility(visible: Boolean) { -// interpreterPane.getComponent().setVisible(visible); -// if( visible ) -// interpreterSplitPane.setDividerLocation(getWindowHeight()/10 * 6); -// else -// interpreterSplitPane.setDividerLocation(getWindowHeight()); - } - - /** - * Create an animation thread with the given fps speed and the specified action - * @param fps frames per second at which this action should be run - * @param action Runnable that contains code to run fps times per second - * @return a Future corresponding to the thread - */ - @Synchronized - fun animate(fps: Int, action: Runnable): Future<*> { - // TODO: Make animation speed less laggy and more accurate. - val delay = 1000 / fps - val thread = threadService.run { - while (animating) { - action.run() - try { - Thread.sleep(delay.toLong()) - } catch (e: InterruptedException) { - break - } - } - } - animations!!.add(thread) - animating = true - return thread - } - - /** - * Stop all animations - */ - @Synchronized - fun stopAnimation() { - animating = false - while (!animations!!.isEmpty()) { - animations!!.peek().cancel(true) - animations!!.remove() - } - } - - /** - * Take a screenshot and save it to the default scenery location - */ - fun takeScreenshot() { - renderer!!.screenshot() - } - - /** - * Take a screenshot and save it to the specified path - * @param path path for saving the screenshot - */ - fun takeScreenshot(path: String?, overwrite: Boolean = false) { - renderer!!.screenshot(path!!, overwrite = overwrite) - } - - /** - * Take a screenshot and return it as an Img - * @return an Img of type UnsignedByteType - */ - val screenshot: Img - get() { - val screenshot = getSceneryRenderer()?.requestScreenshot() ?: throw IllegalStateException("No renderer present, cannot create screenshot") - return ArrayImgs.unsignedBytes(screenshot.data!!, screenshot.width.toLong(), screenshot.height.toLong(), 4L) - } - - /** - * Take a screenshot and return it as an Img - * @return an Img of type UnsignedByteType - */ - val aRGBScreenshot: Img - get() { - return Utils.convertToARGB(screenshot) - } - - /** - * @param name The name of the node to find. - * @return the node object or null, if the node has not been found. - */ - fun find(name: String): Node? { - val n = scene.find(name) - if (n == null) { - logger.warn("Node with name $name not found.") - } - return n - } - - /** - * @return an array of all nodes in the scene except Cameras and PointLights - */ - val sceneNodes: Array - get() = getSceneNodes { n: Node? -> n !is Camera && n !is PointLight } - - /** - * Get a list of nodes filtered by filter predicate - * @param filter, a predicate that filters the candidate nodes - * @return all nodes that match the predicate - */ - fun getSceneNodes(filter: Predicate): Array { - return scene.children.filter{ filter.test(it) }.toTypedArray() - } - - /** - * @return an array of all Node's in the scene - */ - val allSceneNodes: Array - get() = getSceneNodes { _: Node? -> true } - - /** - * Delete the current active node - */ - fun deleteActiveNode() { - deleteNode(activeNode) - } - - /** - * Delete a specified node and control whether the event is published - * @param node node to delete from scene - * @param activePublish whether the deletion should be published - */ - @JvmOverloads - fun deleteNode(node: Node?, activePublish: Boolean = true) { - if(node is Volume) { - node.volumeManager.remove(node) - val toRemove = ArrayList() - for( entry in imageToVolumeMap.entries ) { - if( entry.value == node ) { - toRemove.add(entry.key) - } - } - for(entry in toRemove) { - imageToVolumeMap.remove(entry) - } - } - - for (child in node!!.children) { - deleteNode(child, activePublish) - } - objectService.removeObject(node) - node.parent?.removeChild(node) - if (activeNode == node) { - setActiveNode(null) - } - //maintain consistency - if( activePublish ) { - eventService.publish(NodeRemovedEvent(node)) - } - } - - /** - * Dispose the current scenery renderer, hub, and other scenery things - */ - fun dispose() { - val objs: List = objectService.getObjects(Node::class.java) - for (obj in objs) { - objectService.removeObject(obj) - } - scijavaContext!!.service(SciViewService::class.java).close(this) - close() - // if scijavaContext was not created by ImageJ, then system exit - if( objectService.getObjects(Utils.SciviewStandalone::class.java).size > 0 ) { - log.info("Was running as sciview standalone, shutting down JVM") - System.exit(0) - } - } - - override fun close() { - super.close() - } - - /** - * Move the current active camera to the specified position - * @param position position to move the camera to - */ - fun moveCamera(position: FloatArray) { - camera?.spatial()?.position = Vector3f(position[0], position[1], position[2]) - } - - /** - * Move the current active camera to the specified position - * @param position position to move the camera to - */ - fun moveCamera(position: DoubleArray) { - camera?.spatial()?.position = Vector3f(position[0].toFloat(), position[1].toFloat(), position[2].toFloat()) - } - - /** - * Get the current application name - * @return a String of the application name - */ - fun getName(): String { - return applicationName - } - - /** - * Add a child to the scene. you probably want addNode - * @param node node to add as a child to the scene - */ - fun addChild(node: Node) { - scene.addChild(node) - } - - /** - * Add a Dataset to the scene as a volume. Voxel resolution and name are extracted from the Dataset itself - * @param image image to add as a volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - fun addVolume(image: Dataset, block: Volume.() -> Unit = {}): Volume { - val voxelDims = FloatArray(image.numDimensions()) - - for (d in voxelDims.indices) { - val inValue = image.axis(d).averageScale(0.0, 1.0) - if (image.axis(d).unit() == null) { - voxelDims[d] = inValue.toFloat() - } else { - val imageAxisUnit = image.axis(d).unit().replace("µ", "u") - val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") - - voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() - } - } - - logger.info("Adding with ${voxelDims.joinToString(",")}") - val v = addVolume(image, voxelDims, block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a Dataset as a Volume with the specified voxel dimensions - * @param image image to add as a volume - * @param voxelDimensions dimensions of voxels in volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - @Suppress("UNCHECKED_CAST") - fun addVolume(image: Dataset, voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image.imgPlus as RandomAccessibleInterval>, image.name ?: "Volume", - *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image, name, 1f, 1f, 1f, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - fun > addVolume(image: RandomAccessibleInterval, voxelDimensions: FloatArray, block: Volume.() -> Unit): Volume { - val v = addVolume(image, "volume", *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add an IterableInterval as a Volume - * @param image - * @param - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, "Volume") - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - - /** - * Add an IterableInterval as a Volume - * @param image image to add as a volume - * @param name name of image - * @param pixel type of image - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval, name: String = "Volume"): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, name, 1f, 1f, 1f) - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - - /** - * Set the colormap using an ImageJ LUT name - * @param n node to apply colormap to - * @param lutName name of LUT according to imagej LUTService - */ - fun setColormap(n: Node, lutName: String) { - try { - setColormap(n, lutService.loadLUT(lutService.findLUTs()[lutName])) - } catch (e: IOException) { - e.printStackTrace() - } - } - - /** - * Set the ColorMap of node n to the supplied colorTable - * @param n node to apply colortable to - * @param colorTable ColorTable to use - */ - fun setColormap(n: Node, colorTable: ColorTable) { - val copies = 16 - val byteBuffer = ByteBuffer.allocateDirect( - 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture - val tmp = ByteArray(4 * colorTable.length) - for (k in 0 until colorTable.length) { - for (c in 0 until colorTable.componentCount) { - // TODO this assumes numBits is 8, could be 16 - tmp[4 * k + c] = colorTable[c, k].toByte() - } - if (colorTable.componentCount == 3) { - tmp[4 * k + 3] = 255.toByte() - } - } - for (i in 0 until copies) { - byteBuffer.put(tmp) - } - byteBuffer.flip() - n.metadata["sciviewColormap"] = colorTable - if (n is Volume) { - n.colormap = Colormap.fromColorTable(colorTable) - n.geometryOrNull()?.dirty = true - n.spatial().needsUpdate = true - } - } - - /** - * Adss a SourceAndConverter to the scene. - * - * @param sac The SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - fun > addVolume(sac: SourceAndConverter, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - val sources: MutableList> = ArrayList() - sources.add(sac) - - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) - imageToVolumeMap[sources] = v - imageToVolumeMap[sac] = v - return v - } - - /** - * Add an IterableInterval to the image with the specified voxelDimensions and name - * This version of addVolume does most of the work - * @param image image to add as a volume - * @param name name of image - * @param voxelDimensions dimensions of voxel in volume - * @param pixel type of image - * @return a Node corresponding to the Volume - */ - @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", - vararg voxelDimensions: Float, block: Volume.() -> Unit = {}): Volume { - //log.debug( "Add Volume " + name + " image: " + image ); - val dimensions = LongArray(image.numDimensions()) - image.dimensions(dimensions) - val minPt = LongArray(image.numDimensions()) - - // Get type at min point - val imageRA = image.randomAccess() - image.min(minPt) - imageRA.setPosition(minPt) - val voxelType = imageRA.get()!!.createVariable() - val converterSetups: ArrayList = ArrayList() - val stacks = AxisOrder.splitInputStackIntoSourceStacks(image, AxisOrder.getAxisOrder(AxisOrder.DEFAULT, image, false)) - val sourceTransform = AffineTransform3D() - val sources: ArrayList> = ArrayList>() - var numTimepoints = 1 - for (stack in stacks) { - var s: Source - if (stack.numDimensions() > 3) { - numTimepoints = (stack.max(3) + 1).toInt() - s = RandomAccessibleIntervalSource4D(stack, voxelType, sourceTransform, name) - } else { - s = RandomAccessibleIntervalSource(stack, voxelType, sourceTransform, name) - } - val source = BigDataViewer.wrapWithTransformedSource( - SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) - converterSetups.add(BigDataViewer.createConverterSetup(source, setupId.getAndIncrement())) - sources.add(source) - } - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) - v.metadata.set("RandomAccessibleInterval", image) - imageToVolumeMap[image] = v - return v - } - - /** - * Adds a SourceAndConverter to the scene. - * - * This method actually instantiates the volume. - * - * @param sources The list of SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - @Suppress("UNCHECKED_CAST") - fun > addVolume(sources: List>, - converterSetups: ArrayList, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - var timepoints = numTimepoints - var cacheControl: CacheControl? = null - -// RandomAccessibleInterval image = -// ((RandomAccessibleIntervalSource4D) sources.get(0).getSpimSource()). -// .getSource(0, 0); - val image = sources[0].spimSource.getSource(0, 0) - if (image is VolatileView<*, *>) { - val viewData = (image as VolatileView>).volatileViewData - cacheControl = viewData.cacheControl - } - val dimensions = LongArray(image.numDimensions()) - image.dimensions(dimensions) - val minPt = LongArray(image.numDimensions()) - - // Get type at min point - val imageRA = image.randomAccess() - image.min(minPt) - imageRA.setPosition(minPt) - val voxelType = imageRA.get()!!.createVariable() as T - println("addVolume " + image.numDimensions() + " interval " + image as Interval) - - //int numTimepoints = 1; - if (image.numDimensions() > 3) { - timepoints = image.dimension(3).toInt() - } - val ds = RAISource(voxelType, sources, converterSetups, timepoints, cacheControl) - val options = VolumeViewerOptions() - val v: Volume = RAIVolume(ds, options, hub) - v.name = name - v.metadata["sources"] = sources - v.metadata["VoxelDimensions"] = voxelDimensions - v.spatial().scale = Vector3f(1.0f, voxelDimensions[1]/voxelDimensions[0], voxelDimensions[2]/voxelDimensions[0]) * v.pixelToWorldRatio * 10.0f - val tf = v.transferFunction - val rampMin = 0f - val rampMax = 0.1f - tf.clear() - tf.addControlPoint(0.0f, 0.0f) - tf.addControlPoint(rampMin, 0.0f) - tf.addControlPoint(1.0f, rampMax) - val bg = BoundingGrid() - bg.node = v - - imageToVolumeMap[image] = v - return addNode(v, block = block) - } - - /** - * Adds a SourceAndConverter to the scene. - * - * @param sources The list of SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - fun > addVolume(sources: List>, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - var setupId = 0 - val converterSetups = ArrayList() - for (source in sources) { - converterSetups.add(BigDataViewer.createConverterSetup(source, setupId++)) - } - val v = addVolume(sources, converterSetups, numTimepoints, name, *voxelDimensions, block = block) - imageToVolumeMap[sources] = v - return v - } - - /** - * Get the Volume that corresponds to an image if one exists - * @param image an image of any type (e.g. IterableInterval, RAI, SourceAndConverter) - * @return a Volume corresponding to the input image - */ - fun getVolumeFromImage(image: Any): Volume? { - if( image in imageToVolumeMap ) - return imageToVolumeMap[image] - return null - } - - /** - * Update a volume with the given IterableInterval. - * This method actually populates the volume - * @param image image to update into volume - * @param name name of image - * @param voxelDimensions dimensions of voxel in volume - * @param v existing volume to update - * @param pixel type of image - * @return a Node corresponding to the input volume - */ - @Suppress("UNCHECKED_CAST") - fun > updateVolume(image: IterableInterval, name: String, - voxelDimensions: FloatArray, v: Volume): Volume { - val sacs = v.metadata["sources"] as List>? - val source = sacs!![0].spimSource.getSource(0, 0) // hard coded to timepoint and mipmap 0 - val sCur = Views.iterable(source).cursor() - val iCur = image.cursor() - while (sCur.hasNext()) { - sCur.fwd() - iCur.fwd() - sCur.get()!!.set(iCur.get()) - } - v.name = name - v.metadata["VoxelDimensions"] = voxelDimensions - v.volumeManager.notifyUpdate(v) - v.volumeManager.requestRepaint() - //v.getCacheControls().clear(); - //v.setDirty( true ); - v.spatial().needsUpdate = true - //v.setNeedsUpdateWorld( true ); - return v - } - - /** - * - * @return whether PushMode is currently active - */ - fun getPushMode(): Boolean { - return renderer!!.pushMode - } - - /** - * Set the status of PushMode, which only updates the render panel when there is a change in the scene - * @param push true if push mode should be used - * @return current PushMode status - */ - fun setPushMode(push: Boolean): Boolean { - renderer!!.pushMode = push - return renderer!!.pushMode - } - - protected fun finalize() { - stopAnimation() - } - - fun getScenerySettings(): Settings { - return settings - } - - fun getSceneryStatistics(): Statistics { - return stats - } - - fun getSceneryRenderer(): Renderer? { - return renderer - } - - /** - * Enable VR rendering - */ - fun toggleVRRendering() { - vrActive = !vrActive - val cam = scene.activeObserver as? DetachedHeadCamera ?: return - var ti: TrackerInput? = null - var hmdAdded = false - if (!hub.has(SceneryElement.HMDInput)) { - try { - val hmd = OpenVRHMD(false, true) - if (hmd.initializedAndWorking()) { - hub.add(SceneryElement.HMDInput, hmd) - ti = hmd - } else { - logger.warn("Could not initialise VR headset, just activating stereo rendering.") - } - hmdAdded = true - } catch (e: Exception) { - logger.error("Could not add OpenVRHMD: $e") - } - } else { - ti = hub.getWorkingHMD() - } - if (vrActive && ti != null) { - cam.tracker = ti - } else { - cam.tracker = null - } - renderer!!.pushMode = false - // we need to force reloading the renderer as the HMD might require device or instance extensions - if (renderer is VulkanRenderer && hmdAdded) { - replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) - (renderer as VulkanRenderer).toggleVR() - while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { - logger.debug("Waiting for renderer reinitialisation") - try { - Thread.sleep(200) - } catch (e: InterruptedException) { - e.printStackTrace() - } - } - } else { - renderer!!.toggleVR() - } -// renderer!!.toggleVR() - } - - /** - * Set the rotation of Node N by generating a quaternion from the supplied arguments - * @param n node to set rotation for - * @param x x coord of rotation quat - * @param y y coord of rotation quat - * @param z z coord of rotation quat - * @param w w coord of rotation quat - */ - fun setRotation(n: Node, x: Float, y: Float, z: Float, w: Float) { - n.spatialOrNull()?.rotation = Quaternionf(x, y, z, w) - } - - fun setScale(n: Node, x: Float, y: Float, z: Float) { - n.spatialOrNull()?.scale = Vector3f(x, y, z) - } - - @Suppress("UNUSED_PARAMETER") - fun setColor(n: Node, x: Float, y: Float, z: Float, w: Float) { - val col = Vector3f(x, y, z) - n.ifMaterial { - ambient = col - diffuse = col - specular = col - } - } - - fun setPosition(n: Node, x: Float, y: Float, z: Float) { - n.spatialOrNull()?.position = Vector3f(x, y, z) - } - - fun addWindowListener(wl: WindowListener?) { - (mainWindow as? SwingMainWindow)?.addWindowListener(wl) - } - - override fun axis(i: Int): CalibratedAxis? { - return axes[i] - } - - override fun axes(calibratedAxes: Array) { - axes = calibratedAxes - } - - override fun setAxis(calibratedAxis: CalibratedAxis, i: Int) { - axes[i] = calibratedAxis - } - - override fun realMin(i: Int): Double { - return Double.NEGATIVE_INFINITY - } - - override fun realMin(doubles: DoubleArray) { - for (i in doubles.indices) { - doubles[i] = Double.NEGATIVE_INFINITY - } - } - - override fun realMin(realPositionable: RealPositionable) { - for (i in 0 until realPositionable.numDimensions()) { - realPositionable.move(Double.NEGATIVE_INFINITY, i) - } - } - - override fun realMax(i: Int): Double { - return Double.POSITIVE_INFINITY - } - - override fun realMax(doubles: DoubleArray) { - for (i in doubles.indices) { - doubles[i] = Double.POSITIVE_INFINITY - } - } - - override fun realMax(realPositionable: RealPositionable) { - for (i in 0 until realPositionable.numDimensions()) { - realPositionable.move(Double.POSITIVE_INFINITY, i) - } - } - - override fun numDimensions(): Int { - return axes.size - } - - fun setActiveObserver(screenshotCam: Camera?) { - scene.activeObserver = screenshotCam - } - - fun getActiveObserver(): Camera? { - return scene.activeObserver - } - - /** - * Return a list of all nodes that match a given predicate function - * @param nodeMatchPredicate, returns true if a node is a match - * @return list of nodes that match the predicate - */ - fun findNodes(nodeMatchPredicate: Function1): List { - return scene.discover(scene, nodeMatchPredicate, false) - } - - /* - * Convenience function for getting a string of info about a Node - */ - fun nodeInfoString(n: Node): String { - return "Node name: " + n.name + " Node type: " + n.nodeType + " To String: " + n - } - - /** - * Triggers the inspector tree to be completely rebuilt/refreshed. - */ - fun requestPropEditorRefresh() { - eventService.publish(NodeChangedEvent(scene)) - } - - /** - * Triggers the inspector to rebuild/refresh the given node. - * @param n Root of the subtree to get rebuilt/refreshed. - */ - fun requestPropEditorRefresh(n: Node?) { - eventService.publish(NodeChangedEvent(n)) - } - - fun attachCustomPropertyUIToNode(node: Node, ui: CustomPropertyUI) { - node.metadata["sciview-inspector-${ui.module.info.name}"] = ui - } - - fun getAvailableServices() { - println(scijavaContext!!.serviceIndex) - } - - companion object { - //bounds for the controls - const val FPSSPEED_MINBOUND_SLOW = 0.01f - const val FPSSPEED_MAXBOUND_SLOW = 30.0f - const val FPSSPEED_MINBOUND_FAST = 0.2f - const val FPSSPEED_MAXBOUND_FAST = 600f - const val FPSSPEED_MINBOUND_VERYFAST = 10f - const val FPSSPEED_MAXBOUND_VERYFAST = 2000f - - const val MOUSESPEED_MINBOUND = 0.1f - const val MOUSESPEED_MAXBOUND = 3.0f - const val MOUSESCROLL_MINBOUND = 0.3f - const val MOUSESCROLL_MAXBOUND = 10.0f - - @JvmField - val DEFAULT_COLOR: ColorRGB = Colors.LIGHTGRAY - - /** - * Static launching method - * - * @return a newly created SciView - */ - @JvmStatic - @Throws(Exception::class) - fun create(): SciView { - xinitThreads() - val context = Context(ImageJService::class.java, SciJavaService::class.java, SCIFIOService::class.java, - ThreadService::class.java, ObjectService::class.java, LogService::class.java, MenuService::class.java, - IOService::class.java, EventService::class.java, LUTService::class.java, UnitService::class.java, - DatasetIOService::class.java) - val objectService = context.getService(ObjectService::class.java) - objectService.addObject(Utils.SciviewStandalone()) - val sciViewService = context.service(SciViewService::class.java) - return sciViewService.orCreateActiveSciView - } - - /** - * Static launching method - * DEPRECATED use SciView.create() instead - * - * @return a newly created SciView - */ - @Deprecated("Please use SciView.create() instead.", replaceWith = ReplaceWith("SciView.create()")) - @Throws(Exception::class) - fun createSciView(): SciView { - return create() - } - } -} +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview + +import bdv.BigDataViewer +import bdv.cache.CacheControl +import bdv.tools.brightness.ConverterSetup +import bdv.util.AxisOrder +import bdv.util.RandomAccessibleIntervalSource +import bdv.util.RandomAccessibleIntervalSource4D +import bdv.util.volatiles.VolatileView +import bdv.viewer.Source +import bdv.viewer.SourceAndConverter +import graphics.scenery.* +import graphics.scenery.Scene.RaycastResult +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.opengl.OpenGLRenderer +import graphics.scenery.backends.vulkan.VulkanRenderer +import graphics.scenery.controls.InputHandler +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerInput +import graphics.scenery.primitives.* +import graphics.scenery.proteins.Protein +import graphics.scenery.proteins.RibbonDiagram +import graphics.scenery.utils.ExtractsNatives +import graphics.scenery.utils.ExtractsNatives.Companion.getPlatform +import graphics.scenery.utils.LogbackUtils +import graphics.scenery.utils.SceneryPanel +import graphics.scenery.utils.Statistics +import graphics.scenery.utils.extensions.times +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.Volume.Companion.fromXML +import graphics.scenery.volumes.Volume.Companion.setupId +import graphics.scenery.volumes.Volume.VolumeDataSource.RAISource +import io.scif.SCIFIOService +import io.scif.services.DatasetIOService +import net.imagej.Dataset +import net.imagej.ImageJService +import net.imagej.axis.CalibratedAxis +import net.imagej.axis.DefaultAxisType +import net.imagej.axis.DefaultLinearAxis +import net.imagej.interval.CalibratedRealInterval +import net.imagej.lut.LUTService +import net.imagej.mesh.Mesh +import net.imagej.mesh.io.ply.PLYMeshIO +import net.imagej.mesh.io.stl.STLMeshIO +import net.imagej.units.UnitService +import net.imglib2.* +import net.imglib2.display.ColorTable +import net.imglib2.img.Img +import net.imglib2.img.array.ArrayImgs +import net.imglib2.realtransform.AffineTransform3D +import net.imglib2.type.numeric.ARGBType +import net.imglib2.type.numeric.NumericType +import net.imglib2.type.numeric.RealType +import net.imglib2.type.numeric.integer.UnsignedByteType +import net.imglib2.view.Views +import org.joml.Quaternionf +import org.joml.Vector3f +import org.joml.Vector4f +import org.scijava.Context +import org.scijava.`object`.ObjectService +import org.scijava.display.Display +import org.scijava.event.EventHandler +import org.scijava.event.EventService +import org.scijava.io.IOService +import org.scijava.log.LogLevel +import org.scijava.log.LogService +import org.scijava.menu.MenuService +import org.scijava.plugin.Parameter +import org.scijava.service.SciJavaService +import org.scijava.thread.ThreadService +import org.scijava.util.ColorRGB +import org.scijava.util.Colors +import org.scijava.util.VersionUtils +import sc.iview.commands.demo.animation.ParticleDemo +import sc.iview.event.NodeActivatedEvent +import sc.iview.event.NodeAddedEvent +import sc.iview.event.NodeChangedEvent +import sc.iview.event.NodeRemovedEvent +import sc.iview.process.MeshConverter +import sc.iview.ui.CustomPropertyUI +import sc.iview.ui.MainWindow +import sc.iview.ui.SwingMainWindow +import sc.iview.ui.TaskManager +import tpietzsch.example2.VolumeViewerOptions +import java.awt.event.WindowListener +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.FloatBuffer +import java.nio.file.Path +import java.util.* +import java.util.concurrent.Future +import java.util.function.Consumer +import java.util.function.Function +import java.util.function.Predicate +import java.util.stream.Collectors +import kotlin.collections.ArrayList +import kotlin.collections.HashMap +import kotlin.math.cos +import kotlin.math.sin +import kotlin.system.measureTimeMillis + +/** + * Main SciView class. + * + * @author Kyle Harrington + */ +// we suppress unused warnings here because @Parameter-annotated fields +// get updated automatically by SciJava. +class SciView : SceneryBase, CalibratedRealInterval { + val sceneryPanel = arrayOf(null) + + /* + * Return the default floor object + *//* + * Set the default floor object + */ + /** + * The floor that orients the user in the scene + */ + var floor: Node? = null + protected var vrActive = false + + /** + * The primary camera/observer in the scene + */ + var camera: Camera? = null + set(value) { + field = value + setActiveObserver(field) + } + + lateinit var controls: Controls + val targetArcball: AnimatedCenteringBeforeArcBallControl + get() = controls.targetArcball + + val currentScene: Scene + get() = scene + + /** + * Geometry/Image information of scene + */ + private lateinit var axes: Array + + @Parameter + private lateinit var log: LogService + + @Parameter + private lateinit var menus: MenuService + + @Parameter + private lateinit var io: IOService + + @Parameter + private lateinit var eventService: EventService + + @Parameter + private lateinit var lutService: LUTService + + @Parameter + private lateinit var threadService: ThreadService + + @Parameter + private lateinit var objectService: ObjectService + + @Parameter + private lateinit var unitService: UnitService + + private lateinit var imageToVolumeMap: HashMap + + /** + * Queue keeps track of the currently running animations + */ + private var animations: Queue>? = null + + /** + * Animation pause tracking + */ + private var animating = false + + /** + * This tracks the actively selected Node in the scene + */ + var activeNode: Node? = null + private set + + /* + * Return the SciJava Display that contains SciView + *//* + * Set the SciJava Display + */ var display: Display<*>? = null + + /** + * Return the current SceneryJPanel. This is necessary for custom context menus + * @return panel the current SceneryJPanel + */ + var lights: ArrayList? = null + private set + private val notAbstractNode: Predicate = Predicate { node: Node -> !(node is Camera || node is Light || node === floor) } + var isClosed = false + internal set + + private val notAbstractBranchingFunction = Function { node: Node -> node.children.stream().filter(notAbstractNode).collect(Collectors.toList()) } + + val taskManager = TaskManager() + + // If true, then when a new node is added to the scene, the camera will refocus on this node by default + var centerOnNewNodes = false + + // If true, then when a new node is added the thread will block until the node is added to the scene. This is required for + // centerOnNewNodes + var blockOnNewNodes = false + private var headlight: PointLight? = null + + lateinit var mainWindow: MainWindow + + constructor(context: Context) : super("SciView", 1280, 720, false, context) { + context.inject(this) + } + + constructor(applicationName: String?, windowWidth: Int, windowHeight: Int) : super(applicationName!!, windowWidth, windowHeight, false) + + fun publicGetInputHandler(): InputHandler { + return inputHandler!! + } + + /** + * Toggle video recording with scenery's video recording mechanism + * Note: this video recording may skip frames because it is asynchronous + */ + fun toggleRecordVideo() { + if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie() else (renderer as VulkanRenderer).recordMovie() + } + + /** + * Toggle video recording with scenery's video recording mechanism + * Note: this video recording may skip frames because it is asynchronous + * + * @param filename destination for saving video + * @param overwrite should the file be replaced, otherwise a unique incrementing counter will be appended + */ + fun toggleRecordVideo(filename: String?, overwrite: Boolean) { + if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie(filename!!, overwrite) else (renderer as VulkanRenderer).recordMovie(filename!!, overwrite) + } + + /** + * See [Controls.stashControls]. + */ + fun stashControls() { + controls.stashControls() + } + + /** + * See [Controls.restoreControls] and [Controls.stashControls]. + */ + fun restoreControls() { + controls.restoreControls() + } + + internal fun setRenderer(newRenderer: Renderer) { + renderer = newRenderer + } + + + /** + * Reset the scene to initial conditions + */ + fun reset() { + // Initialize the 3D axes + axes = arrayOf( + DefaultLinearAxis(DefaultAxisType("X", true), "um", 1.0), + DefaultLinearAxis(DefaultAxisType("Y", true), "um", 1.0), + DefaultLinearAxis(DefaultAxisType("Z", true), "um", 1.0) + ) + + // Remove everything except camera + val toRemove = getSceneNodes { n: Node? -> n !is Camera } + for (n in toRemove) { + deleteNode(n, false) + } + + imageToVolumeMap = HashMap() + + // Setup camera + if (camera == null) { + camera = DetachedHeadCamera() + (camera as DetachedHeadCamera).position = Vector3f(0.0f, 1.65f, 0.0f) + scene.addChild(camera as DetachedHeadCamera) + } + camera!!.spatial().position = Vector3f(0.0f, 1.65f, 5.0f) + camera!!.perspectiveCamera(50.0f, windowWidth, windowHeight, 0.1f, 1000.0f) + + // Setup lights + val tetrahedron = arrayOfNulls(4) + tetrahedron[0] = Vector3f(1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[1] = Vector3f(-1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[2] = Vector3f(0.0f, 1.0f, 1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[3] = Vector3f(0.0f, -1.0f, 1.0f / Math.sqrt(2.0).toFloat()) + lights = ArrayList() + for (i in 0..3) { // TODO allow # initial lights to be customizable? + val light = PointLight(150.0f) + light.spatial().position = tetrahedron[i]!!.mul(25.0f) + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + light.intensity = 1.0f + lights!!.add(light) + //camera.addChild( light ); + scene.addChild(light) + } + + // Make a headlight for the camera + headlight = PointLight(150.0f) + headlight!!.spatial().position = Vector3f(0f, 0f, -1f).mul(25.0f) + headlight!!.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + headlight!!.intensity = 0.5f + headlight!!.name = "headlight" + val lightSphere = Icosphere(1.0f, 2) + headlight!!.addChild(lightSphere) + lightSphere.material().diffuse = headlight!!.emissionColor + lightSphere.material().specular = headlight!!.emissionColor + lightSphere.material().ambient = headlight!!.emissionColor + lightSphere.material().wireframe = true + lightSphere.visible = false + //lights.add( light ); + camera!!.nearPlaneDistance = 0.01f + camera!!.farPlaneDistance = 1000.0f + camera!!.addChild(headlight!!) + floor = InfinitePlane() //new Box( new Vector3f( 500f, 0.2f, 500f ) ); + (floor as InfinitePlane).type = InfinitePlane.Type.Grid + (floor as Node).name = "Floor" + scene.addChild(floor as Node) + } + + /** + * Initialization of SWING and scenery. Also triggers an initial population of lights/camera in the scene + */ + override fun init() { + val logLevel = System.getProperty("scenery.LogLevel", "info") + log.level = LogLevel.value(logLevel) + LogbackUtils.setLogLevel(null, logLevel) + System.getProperties().stringPropertyNames().forEach(Consumer { name: String -> + if (name.startsWith("scenery.LogLevel")) { + LogbackUtils.setLogLevel("", System.getProperty(name, "info")) + } + }) + + // determine imagej-launcher version and to disable Vulkan if XInitThreads() fix + // is not deployed + try { + val launcherClass = Class.forName("net.imagej.launcher.ClassLauncher") + var versionString = VersionUtils.getVersion(launcherClass) + if (versionString != null && getPlatform() == ExtractsNatives.Platform.LINUX) { + versionString = versionString.substring(0, 5) + val launcherVersion = Version(versionString) + val nonWorkingVersion = Version("4.0.5") + if (launcherVersion.compareTo(nonWorkingVersion) <= 0 + && !java.lang.Boolean.parseBoolean(System.getProperty("sciview.DisableLauncherVersionCheck", "false"))) { + logger.info("imagej-launcher version smaller or equal to non-working version ($versionString vs. 4.0.5), disabling Vulkan as rendering backend. Disable check by setting 'scenery.DisableLauncherVersionCheck' system property to 'true'.") + System.setProperty("scenery.Renderer", "OpenGLRenderer") + } else { + logger.info("imagej-launcher version bigger that non-working version ($versionString vs. 4.0.5), all good.") + } + } + } catch (cnfe: ClassNotFoundException) { + // Didn't find the launcher, so we're probably good. + logger.info("imagej-launcher not found, not touching renderer preferences.") + } + + animations = LinkedList() + mainWindow = SwingMainWindow(this) + controls = Controls(this) + + imageToVolumeMap = HashMap() + } + + fun toggleSidebar(): Boolean { + return mainWindow.toggleSidebar() + + } + + private fun initializeInterpreter() { + mainWindow.initializeInterpreter() + } + + /* + * Completely close the SciView window + cleanup + */ + fun closeWindow() { + mainWindow.close() + dispose() + } + + /* + * Return true if the scene has been initialized + */ + val isInitialized: Boolean + get() = sceneInitialized() + + /** + * Place the scene into the center of camera view, and zoom in/out such + * that the whole scene is in the view (everything would be visible if it + * would not be potentially occluded). + */ + fun fitCameraToScene() { + centerOnNode(scene) + //TODO: smooth zoom in/out VLADO vlado Vlado + } + + /** + * Place the scene into the center of camera view. + */ + fun centerOnScene() { + centerOnNode(scene) + } + /* + * Get the InputHandler that is managing mouse, input, VR controls, etc. + */ + val sceneryInputHandler: InputHandler + get() = inputHandler!! + + /* + * Return a bounding box around a subgraph of the scenegraph + */ + fun getSubgraphBoundingBox(n: Node): OrientedBoundingBox? { + val predicate = Function> { node: Node -> node.children } + return getSubgraphBoundingBox(n, predicate) + } + + /* + * Return a bounding box around a subgraph of the scenegraph + */ + fun getSubgraphBoundingBox(n: Node, branchFunction: Function>): OrientedBoundingBox? { + if (n.boundingBox == null && n.children.size != 0) { + return n.getMaximumBoundingBox().asWorld() + } + val branches = branchFunction.apply(n) + if (branches.isEmpty()) { + return if (n.boundingBox == null) null else n.boundingBox!!.asWorld() + } + var bb = n.getMaximumBoundingBox() + for (c in branches) { + val cBB = getSubgraphBoundingBox(c, branchFunction) + if (cBB != null) bb = bb.expand(bb, cBB) + } + return bb + } + + /** + * Place the active node into the center of camera view. + */ + fun centerOnActiveNode() { + if (activeNode == null) return + centerOnNode(activeNode) + } + + /** + * Place the specified node into the center of camera view. + */ + fun centerOnNode(currentNode: Node?) { + if (currentNode == null) { + log.info("Cannot center on null node.") + return + } + + //center the on the same spot as ArcBall does + centerOnPosition(currentNode.getMaximumBoundingBox().getBoundingSphere().origin) + } + + /** + * Center the camera on the specified Node + */ + fun centerOnPosition(currentPos: Vector3f?) { + controls.centerOnPosition(currentPos) + } + + /** + * Activate the node, and center the view on it. + * @param n + * @return the currently active node + */ + fun setActiveCenteredNode(n: Node?): Node? { + //activate... + val ret = setActiveNode(n) + //...and center it + ret?.let { centerOnNode(it) } + return ret + } + + //a couple of shortcut methods to readout controls params + fun getFPSSpeedSlow(): Float { + return controls.getFPSSpeedSlow() + } + + fun getFPSSpeedFast(): Float { + return controls.getFPSSpeedFast() + } + + fun getFPSSpeedVeryFast(): Float { + return controls.getFPSSpeedVeryFast() + } + + fun getMouseSpeed(): Float { + return controls.getMouseSpeed() + } + + fun getMouseScrollSpeed(): Float { + return controls.getMouseScrollSpeed() + } + + //a couple of setters with scene sensible boundary checks + fun setFPSSpeedSlow(slowSpeed: Float) { + controls.setFPSSpeedSlow(slowSpeed) + } + + fun setFPSSpeedFast(fastSpeed: Float) { + controls.setFPSSpeedFast(fastSpeed) + } + + fun setFPSSpeedVeryFast(veryFastSpeed: Float) { + controls.setFPSSpeedVeryFast(veryFastSpeed) + } + + fun setFPSSpeed(newBaseSpeed: Float) { + controls.setFPSSpeed(newBaseSpeed) + } + + fun setMouseSpeed(newSpeed: Float) { + controls.setMouseSpeed(newSpeed) + } + + fun setMouseScrollSpeed(newSpeed: Float) { + controls.setMouseScrollSpeed(newSpeed) + } + + fun setObjectSelectionMode() { + controls.setObjectSelectionMode() + } + + /* + * Set the action used during object selection + */ + fun setObjectSelectionMode(selectAction: Function3?) { + controls.setObjectSelectionMode(selectAction) + } + + fun showContextNodeChooser(x: Int, y: Int) { + mainWindow.showContextNodeChooser(x,y) + } + + /* + * Initial configuration of the scenery InputHandler + * This is automatically called and should not be used directly + */ + override fun inputSetup() { + log.info("Running InputSetup") + controls.inputSetup() + } + + /** + * Add a box at the specified position with specified size, color, and normals on the inside/outside + * @param position position to put the box + * @param size size of the box + * @param color color of the box + * @param inside are normals inside the box? + * @return the Node corresponding to the box + */ + @JvmOverloads + fun addBox(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), size: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, + inside: Boolean = false, block: Box.() -> Unit = {}): Box { + val box = Box(size, inside) + box.spatial().position = position + box.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + return addNode(box, block = block) + } + + /** + * Add a unit sphere at a given [position] with given [radius] and [color]. + * @return the Node corresponding to the sphere + */ + @JvmOverloads + fun addSphere(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), radius: Float = 1f, color: ColorRGB = DEFAULT_COLOR, block: Sphere.() -> Unit = {}): Sphere { + val sphere = Sphere(radius, 20) + sphere.spatial().position = position + sphere.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + + return addNode(sphere, block = block) + } + + /** + * Add a Cylinder at the given position with radius, height, and number of faces/segments + * @param position position of the cylinder + * @param radius radius of the cylinder + * @param height height of the cylinder + * @param num_segments number of segments to represent the cylinder + * @return the Node corresponding to the cylinder + */ + fun addCylinder(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cylinder.() -> Unit = {}): Cylinder { + val cyl = Cylinder(radius, height, num_segments) + cyl.spatial().position = position + return addNode(cyl, block = block) + } + + /** + * Add a Cone at the given position with radius, height, and number of faces/segments + * @param position position to put the cone + * @param radius radius of the cone + * @param height height of the cone + * @param num_segments number of segments used to represent cone + * @return the Node corresponding to the cone + */ + fun addCone(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cone.() -> Unit = {}): Cone { + val cone = Cone(radius, height, num_segments, Vector3f(0.0f, 0.0f, 1.0f)) + cone.spatial().position = position + return addNode(cone, block = block) + } + + /** + * Add a line from start to stop with the given color + * @param start start position of line + * @param stop stop position of line + * @param color color of line + * @return the Node corresponding to the line + */ + @JvmOverloads + fun addLine(start: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), stop: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, block: Line.() -> Unit = {}): Line { + return addLine(arrayOf(start, stop), color, 0.1, block) + } + + /** + * Add a multi-segment line that goes through the supplied points with a single color and edge width + * @param points points along line including first and terminal points + * @param color color of line + * @param edgeWidth width of line segments + * @return the Node corresponding to the line + */ + @JvmOverloads + fun addLine(points: Array, color: ColorRGB, edgeWidth: Double, block: Line.() -> Unit = {}): Line { + val line = Line(points.size) + for (pt in points) { + line.addPoint(pt) + } + line.edgeWidth = edgeWidth.toFloat() + line.material { + ambient = Vector3f(1.0f, 1.0f, 1.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + line.spatial().position = points[0] + return addNode(line, block = block) + } + + /** + * Add a PointLight source at the origin + * @return a Node corresponding to the PointLight + */ + @JvmOverloads + fun addPointLight(block: PointLight.() -> Unit = {}): PointLight { + val light = PointLight(5.0f) + light.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Vector3f(0.0f, 1.0f, 0.0f) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + light.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + lights!!.add(light) + return addNode(light, block = block) + } + + /** + * Position all lights that were initialized by default around the scene in a circle at Y=0 + */ + fun surroundLighting() { + val bb = getSubgraphBoundingBox(scene, notAbstractBranchingFunction) + val (c, r) = bb!!.getBoundingSphere() + // Choose a good y-position, then place lights around the cross-section through this plane + val y = 0f + for (k in lights!!.indices) { + val light = lights!![k] + val x = (c.x() + r * cos(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() + val z = (c.y() + r * sin(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() + light.lightRadius = 2 * r + light.spatial().position = Vector3f(x, y, z) + } + } + + @Throws(IOException::class) + fun openDirTiff(source: Path) + { + val v = Volume.fromPath(source, hub) + v.name = "volume" + v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = true + + v.spatial().wantsComposeModel = true + v.spatial().updateWorld(true) + System.out.println("v.model: " + v.model) + addChild(v) + System.out.println("v.getDimensions: "+ v.getDimensions()) + + System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) + System.out.println("v.world.matrix: " + v.spatial().world) + } + + data class PointInTrack( + val t: Int, + val loc: Vector3f, + val cellId: Long, + val parentId: Long, + val nodeScore: Float, + val edgeScore: Float + ) + + data class Track( + val track: List, + val trackId: Int + ) + + @Throws(IOException::class) + fun openTrackFile(file: File) + { + val lines = file.readLines() + var track = ArrayList() + val tracks = ArrayList() + val separator = "," + + var lastTrackId = -1 + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val t = tokens[0].toInt() + val z = tokens[1].toFloat() -2000f + val y = tokens[2].toFloat() -800f + val x = tokens[3].toFloat() -1300f + val cellId = tokens[4].toLong() + val parentId = tokens[5].toLong() + val trackId = tokens[6].toInt() + val nodeScore = tokens[7].toFloat() + val edgeScore = tokens[8].toFloat()/45.0f + + val currentPointInTrack = PointInTrack( + t, + Vector3f(x,y,z), + cellId, + parentId, + nodeScore, + edgeScore + ) + if(lastTrackId != trackId) + { + lastTrackId = trackId + val sortedTrack = track.sortedBy { it.t } + tracks.add(Track(sortedTrack, trackId)) + + track.clear() + } + track.add(currentPointInTrack) + } + val timeCost = measureTimeMillis { + addTracks(tracks) + } + println("time: $timeCost") + } + + fun addTracks(tracks: ArrayList) + { + val rng = Random(17) + for(track in tracks) + { + if(track.trackId > 10) + { + // continue + } + System.out.println("add track: "+ track.trackId.toString() ) + val master = Cylinder(0.1f, 1.0f, 10) +// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Vector3f(0.05f, 0f, 0f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + mInstanced.name = "TrackID-${track.trackId}" + mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } + addNode(mInstanced) + + var cnt = 0 + val a = rng.nextFloat() + val b = rng.nextFloat() + track.track.windowed(2,1).forEach { pair -> + cnt = cnt + 1 + val element = mInstanced.addInstance() + element.name ="EdgeID-$cnt" + element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} + element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) + //mInstanced.instances.add(element) + + } + + } + + } + + + /** + * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud + * @param source string of a data source + * @throws IOException + */ + @Suppress("UNCHECKED_CAST") + @Throws(IOException::class) + fun open(source: String) { + if (source.endsWith(".xml", ignoreCase = true)) { + addNode(fromXML(source, hub, VolumeViewerOptions())) + return + } else if (source.takeLast(4).equals(".pdb", true)) { + val protein = Protein.fromFile(source) + val ribbon = RibbonDiagram(protein) + ribbon.spatial().position = Vector3f(0f, 0f, 0f) + addNode(ribbon) + return + } else if (source.endsWith(".stl", ignoreCase = true)) { + val stlReader = STLMeshIO() + addMesh(stlReader.open(source)) + return + } else if (source.endsWith(".ply", ignoreCase = true)) { + val plyReader = PLYMeshIO() + addMesh(plyReader.open(source)) + return + } + val data = io.open(source) + if (data is Mesh) + addMesh(data) + else if (data is graphics.scenery.Mesh) + addMesh(data) + else if (data is PointCloud) + addPointCloud(data) + else if (data is Dataset) + addVolume(data) + else if (data is RandomAccessibleInterval<*>) + addVolume(data as RandomAccessibleInterval>, source) + else if (data is List<*>) { + val list = data + require(!list.isEmpty()) { "Data source '$source' appears empty." } + val element = list[0] + if (element is RealLocalizable) { + // NB: For now, we assume all elements will be RealLocalizable. + // Highly likely to be the case, barring antagonistic importers. + val points = list as List + addPointCloud(points, source) + } else { + val type = if (element == null) "" else element.javaClass.name + throw IllegalArgumentException("Data source '" + source + // + "' contains elements of unknown type '" + type + "'") + } + } else { + val type = if (data == null) "" else data.javaClass.name + throw IllegalArgumentException("Data source '" + source + // + "' contains data of unknown type '" + type + "'") + } + } + + /** + * Add the given points to the scene as a PointCloud with a given name + * @param points points to use in a PointCloud + * @param name name of the PointCloud + * @return + */ + @JvmOverloads + fun addPointCloud(points: Collection, + name: String? = "PointCloud", + pointSize : Float = 1.0f, + block: PointCloud.() -> Unit = {}): PointCloud { + val flatVerts = FloatArray(points.size * 3) + var k = 0 + for (point in points) { + flatVerts[k * 3] = point.getFloatPosition(0) + flatVerts[k * 3 + 1] = point.getFloatPosition(1) + flatVerts[k * 3 + 2] = point.getFloatPosition(2) + k++ + } + val pointCloud = PointCloud(pointSize, name!!) + val vBuffer: FloatBuffer = BufferUtils.allocateFloat(flatVerts.size * 4) + val nBuffer: FloatBuffer = BufferUtils.allocateFloat(0) + vBuffer.put(flatVerts) + vBuffer.flip() + pointCloud.geometry().vertices = vBuffer + pointCloud.geometry().normals = nBuffer + pointCloud.geometry().indices = BufferUtils.allocateInt(0) + pointCloud.spatial().position = Vector3f(0f, 0f, 0f) + + pointCloud.setupPointCloud() + return addNode(pointCloud, block = block) + } + + /** + * Add a PointCloud to the scene + * @param pointCloud existing PointCloud to add to scene + * @return a Node corresponding to the PointCloud + */ + @JvmOverloads + fun addPointCloud(pointCloud: PointCloud, block: PointCloud.() -> Unit = {}): PointCloud { + pointCloud.setupPointCloud() + pointCloud.spatial().position = Vector3f(0f, 0f, 0f) + return addNode(pointCloud, block = block) + } + + /** + * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true + * @param n node to add to scene + * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services + * @return a Node corresponding to the Node + */ + @JvmOverloads + fun addNode(n: N, activePublish: Boolean = true, block: N.() -> Unit = {}): N { + n?.let { + it.block() + scene.addChild(it) + objectService.addObject(n) + if (blockOnNewNodes) { + Utils.blockWhile({ this.find(n.name) == null }, 20) + //System.out.println("find(name) " + find(n.getName()) ); + } + // Set new node as active and centered? + setActiveNode(n) + if (centerOnNewNodes) { + centerOnNode(n) + } + if (activePublish) { + eventService.publish(NodeAddedEvent(n)) + } + } + return n + } + + /** + * Add a scenery Mesh to the scene + * @param scMesh scenery mesh to add to scene + * @return a Node corresponding to the mesh + */ + fun addMesh(scMesh: graphics.scenery.Mesh): graphics.scenery.Mesh { + scMesh.ifMaterial { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Vector3f(0.0f, 1.0f, 0.0f) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + scMesh.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + objectService.addObject(scMesh) + return addNode(scMesh) + } + + /** + * Add an ImageJ mesh to the scene + * @param mesh net.imagej.mesh to add to scene + * @return a Node corresponding to the mesh + */ + fun addMesh(mesh: Mesh): graphics.scenery.Mesh { + val scMesh = MeshConverter.toScenery(mesh) + return addMesh(scMesh) + } + + /** + * [Deprecated: use deleteNode] + * Remove a Mesh from the scene + * @param scMesh mesh to remove from scene + */ + fun removeMesh(scMesh: graphics.scenery.Mesh?) { + scene.removeChild(scMesh!!) + } + + /** + * Set the currently active node + * @param n existing node that should become active focus of this SciView + * @return the currently active node + */ + fun setActiveNode(n: Node?): Node? { + if (activeNode === n) return activeNode + activeNode = n + targetArcball.target = { n?.getMaximumBoundingBox()?.getBoundingSphere()?.origin ?: Vector3f(0.0f, 0.0f, 0.0f) } + mainWindow.selectNode(activeNode) + eventService.publish(NodeActivatedEvent(activeNode)) + return activeNode + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeAdded(event: NodeAddedEvent?) { + mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeRemoved(event: NodeRemovedEvent?) { + mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeChanged(event: NodeChangedEvent?) { + // TODO: Check if rebuilding the tree is necessary here, otherwise this costs a lot of performance + //mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeActivated(event: NodeActivatedEvent?) { + // TODO: add listener code for node activation, if necessary + // NOTE: do not update property window here, this will lead to a loop. + } + + fun toggleInspectorWindow() { + toggleSidebar() + } + + @Suppress("UNUSED_PARAMETER") + fun setInspectorWindowVisibility(visible: Boolean) { +// inspector.setVisible(visible); +// if( visible ) +// mainSplitPane.setDividerLocation(getWindowWidth()/4 * 3); +// else +// mainSplitPane.setDividerLocation(getWindowWidth()); + } + + @Suppress("UNUSED_PARAMETER") + fun setInterpreterWindowVisibility(visible: Boolean) { +// interpreterPane.getComponent().setVisible(visible); +// if( visible ) +// interpreterSplitPane.setDividerLocation(getWindowHeight()/10 * 6); +// else +// interpreterSplitPane.setDividerLocation(getWindowHeight()); + } + + /** + * Create an animation thread with the given fps speed and the specified action + * @param fps frames per second at which this action should be run + * @param action Runnable that contains code to run fps times per second + * @return a Future corresponding to the thread + */ + @Synchronized + fun animate(fps: Int, action: Runnable): Future<*> { + // TODO: Make animation speed less laggy and more accurate. + val delay = 1000 / fps + val thread = threadService.run { + while (animating) { + action.run() + try { + Thread.sleep(delay.toLong()) + } catch (e: InterruptedException) { + break + } + } + } + animations!!.add(thread) + animating = true + return thread + } + + /** + * Stop all animations + */ + @Synchronized + fun stopAnimation() { + animating = false + while (!animations!!.isEmpty()) { + animations!!.peek().cancel(true) + animations!!.remove() + } + } + + /** + * Take a screenshot and save it to the default scenery location + */ + fun takeScreenshot() { + renderer!!.screenshot() + } + + /** + * Take a screenshot and save it to the specified path + * @param path path for saving the screenshot + */ + fun takeScreenshot(path: String?, overwrite: Boolean = false) { + renderer!!.screenshot(path!!, overwrite = overwrite) + } + + /** + * Take a screenshot and return it as an Img + * @return an Img of type UnsignedByteType + */ + val screenshot: Img + get() { + val screenshot = getSceneryRenderer()?.requestScreenshot() ?: throw IllegalStateException("No renderer present, cannot create screenshot") + return ArrayImgs.unsignedBytes(screenshot.data!!, screenshot.width.toLong(), screenshot.height.toLong(), 4L) + } + + /** + * Take a screenshot and return it as an Img + * @return an Img of type UnsignedByteType + */ + val aRGBScreenshot: Img + get() { + return Utils.convertToARGB(screenshot) + } + + /** + * @param name The name of the node to find. + * @return the node object or null, if the node has not been found. + */ + fun find(name: String): Node? { + val n = scene.find(name) + if (n == null) { + logger.warn("Node with name $name not found.") + } + return n + } + + /** + * @return an array of all nodes in the scene except Cameras and PointLights + */ + val sceneNodes: Array + get() = getSceneNodes { n: Node? -> n !is Camera && n !is PointLight } + + /** + * Get a list of nodes filtered by filter predicate + * @param filter, a predicate that filters the candidate nodes + * @return all nodes that match the predicate + */ + fun getSceneNodes(filter: Predicate): Array { + return scene.children.filter{ filter.test(it) }.toTypedArray() + } + + /** + * @return an array of all Node's in the scene + */ + val allSceneNodes: Array + get() = getSceneNodes { _: Node? -> true } + + /** + * Delete the current active node + */ + fun deleteActiveNode() { + deleteNode(activeNode) + } + + /** + * Delete a specified node and control whether the event is published + * @param node node to delete from scene + * @param activePublish whether the deletion should be published + */ + @JvmOverloads + fun deleteNode(node: Node?, activePublish: Boolean = true) { + if(node is Volume) { + node.volumeManager.remove(node) + val toRemove = ArrayList() + for( entry in imageToVolumeMap.entries ) { + if( entry.value == node ) { + toRemove.add(entry.key) + } + } + for(entry in toRemove) { + imageToVolumeMap.remove(entry) + } + } + + for (child in node!!.children) { + deleteNode(child, activePublish) + } + objectService.removeObject(node) + node.parent?.removeChild(node) + if (activeNode == node) { + setActiveNode(null) + } + //maintain consistency + if( activePublish ) { + eventService.publish(NodeRemovedEvent(node)) + } + } + + /** + * Dispose the current scenery renderer, hub, and other scenery things + */ + fun dispose() { + val objs: List = objectService.getObjects(Node::class.java) + for (obj in objs) { + objectService.removeObject(obj) + } + scijavaContext!!.service(SciViewService::class.java).close(this) + close() + // if scijavaContext was not created by ImageJ, then system exit + if( objectService.getObjects(Utils.SciviewStandalone::class.java).size > 0 ) { + log.info("Was running as sciview standalone, shutting down JVM") + System.exit(0) + } + } + + override fun close() { + super.close() + } + + /** + * Move the current active camera to the specified position + * @param position position to move the camera to + */ + fun moveCamera(position: FloatArray) { + camera?.spatial()?.position = Vector3f(position[0], position[1], position[2]) + } + + /** + * Move the current active camera to the specified position + * @param position position to move the camera to + */ + fun moveCamera(position: DoubleArray) { + camera?.spatial()?.position = Vector3f(position[0].toFloat(), position[1].toFloat(), position[2].toFloat()) + } + + /** + * Get the current application name + * @return a String of the application name + */ + fun getName(): String { + return applicationName + } + + /** + * Add a child to the scene. you probably want addNode + * @param node node to add as a child to the scene + */ + fun addChild(node: Node) { + scene.addChild(node) + } + + /** + * Add a Dataset to the scene as a volume. Voxel resolution and name are extracted from the Dataset itself + * @param image image to add as a volume + * @return a Node corresponding to the Volume + */ + @JvmOverloads + fun addVolume(image: Dataset, block: Volume.() -> Unit = {}): Volume { + val voxelDims = FloatArray(image.numDimensions()) + + for (d in voxelDims.indices) { + val inValue = image.axis(d).averageScale(0.0, 1.0) + if (image.axis(d).unit() == null) { + voxelDims[d] = inValue.toFloat() + } else { + val imageAxisUnit = image.axis(d).unit().replace("µ", "u") + val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") + + voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() + } + } + + logger.info("Adding with ${voxelDims.joinToString(",")}") + val v = addVolume(image, voxelDims, block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a Dataset as a Volume with the specified voxel dimensions + * @param image image to add as a volume + * @param voxelDimensions dimensions of voxels in volume + * @return a Node corresponding to the Volume + */ + @JvmOverloads + @Suppress("UNCHECKED_CAST") + fun addVolume(image: Dataset, voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { + val v = addVolume(image.imgPlus as RandomAccessibleInterval>, image.name ?: "Volume", + *voxelDimensions, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a RandomAccessibleInterval to the image + * @param image image to add as a volume + * @param pixel type of image + * @return a Node corresponding to the volume + */ + @JvmOverloads + fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", block: Volume.() -> Unit = {}): Volume { + val v = addVolume(image, name, 1f, 1f, 1f, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a RandomAccessibleInterval to the image + * @param image image to add as a volume + * @param pixel type of image + * @return a Node corresponding to the volume + */ + fun > addVolume(image: RandomAccessibleInterval, voxelDimensions: FloatArray, block: Volume.() -> Unit): Volume { + val v = addVolume(image, "volume", *voxelDimensions, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add an IterableInterval as a Volume + * @param image + * @param + * @return a Node corresponding to the Volume + */ + @Suppress("UNCHECKED_CAST") + @Throws(Exception::class) + fun > addVolume(image: IterableInterval): Volume { + return if (image is RandomAccessibleInterval<*>) { + val v = addVolume(image as RandomAccessibleInterval>, "Volume") + imageToVolumeMap[image] = v + v + } else { + throw Exception("Unsupported Volume type:$image") + } + } + + /** + * Add an IterableInterval as a Volume + * @param image image to add as a volume + * @param name name of image + * @param pixel type of image + * @return a Node corresponding to the Volume + */ + @Suppress("UNCHECKED_CAST") + @Throws(Exception::class) + fun > addVolume(image: IterableInterval, name: String = "Volume"): Volume { + return if (image is RandomAccessibleInterval<*>) { + val v = addVolume(image as RandomAccessibleInterval>, name, 1f, 1f, 1f) + imageToVolumeMap[image] = v + v + } else { + throw Exception("Unsupported Volume type:$image") + } + } + + /** + * Set the colormap using an ImageJ LUT name + * @param n node to apply colormap to + * @param lutName name of LUT according to imagej LUTService + */ + fun setColormap(n: Node, lutName: String) { + try { + setColormap(n, lutService.loadLUT(lutService.findLUTs()[lutName])) + } catch (e: IOException) { + e.printStackTrace() + } + } + + /** + * Set the ColorMap of node n to the supplied colorTable + * @param n node to apply colortable to + * @param colorTable ColorTable to use + */ + fun setColormap(n: Node, colorTable: ColorTable) { + val copies = 16 + val byteBuffer = ByteBuffer.allocateDirect( + 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture + val tmp = ByteArray(4 * colorTable.length) + for (k in 0 until colorTable.length) { + for (c in 0 until colorTable.componentCount) { + // TODO this assumes numBits is 8, could be 16 + tmp[4 * k + c] = colorTable[c, k].toByte() + } + if (colorTable.componentCount == 3) { + tmp[4 * k + 3] = 255.toByte() + } + } + for (i in 0 until copies) { + byteBuffer.put(tmp) + } + byteBuffer.flip() + n.metadata["sciviewColormap"] = colorTable + if (n is Volume) { + n.colormap = Colormap.fromColorTable(colorTable) + n.geometryOrNull()?.dirty = true + n.spatial().needsUpdate = true + } + } + + /** + * Adss a SourceAndConverter to the scene. + * + * @param sac The SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + fun > addVolume(sac: SourceAndConverter, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + val sources: MutableList> = ArrayList() + sources.add(sac) + + val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) + imageToVolumeMap[sources] = v + imageToVolumeMap[sac] = v + return v + } + + /** + * Add an IterableInterval to the image with the specified voxelDimensions and name + * This version of addVolume does most of the work + * @param image image to add as a volume + * @param name name of image + * @param voxelDimensions dimensions of voxel in volume + * @param pixel type of image + * @return a Node corresponding to the Volume + */ + @JvmOverloads + fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", + vararg voxelDimensions: Float, block: Volume.() -> Unit = {}): Volume { + //log.debug( "Add Volume " + name + " image: " + image ); + val dimensions = LongArray(image.numDimensions()) + image.dimensions(dimensions) + val minPt = LongArray(image.numDimensions()) + + // Get type at min point + val imageRA = image.randomAccess() + image.min(minPt) + imageRA.setPosition(minPt) + val voxelType = imageRA.get()!!.createVariable() + val converterSetups: ArrayList = ArrayList() + val stacks = AxisOrder.splitInputStackIntoSourceStacks(image, AxisOrder.getAxisOrder(AxisOrder.DEFAULT, image, false)) + val sourceTransform = AffineTransform3D() + val sources: ArrayList> = ArrayList>() + var numTimepoints = 1 + for (stack in stacks) { + var s: Source + if (stack.numDimensions() > 3) { + numTimepoints = (stack.max(3) + 1).toInt() + s = RandomAccessibleIntervalSource4D(stack, voxelType, sourceTransform, name) + } else { + s = RandomAccessibleIntervalSource(stack, voxelType, sourceTransform, name) + } + val source = BigDataViewer.wrapWithTransformedSource( + SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) + converterSetups.add(BigDataViewer.createConverterSetup(source, setupId.getAndIncrement())) + sources.add(source) + } + val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) + v.metadata.set("RandomAccessibleInterval", image) + imageToVolumeMap[image] = v + return v + } + + /** + * Adds a SourceAndConverter to the scene. + * + * This method actually instantiates the volume. + * + * @param sources The list of SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + @Suppress("UNCHECKED_CAST") + fun > addVolume(sources: List>, + converterSetups: ArrayList, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + var timepoints = numTimepoints + var cacheControl: CacheControl? = null + +// RandomAccessibleInterval image = +// ((RandomAccessibleIntervalSource4D) sources.get(0).getSpimSource()). +// .getSource(0, 0); + val image = sources[0].spimSource.getSource(0, 0) + if (image is VolatileView<*, *>) { + val viewData = (image as VolatileView>).volatileViewData + cacheControl = viewData.cacheControl + } + val dimensions = LongArray(image.numDimensions()) + image.dimensions(dimensions) + val minPt = LongArray(image.numDimensions()) + + // Get type at min point + val imageRA = image.randomAccess() + image.min(minPt) + imageRA.setPosition(minPt) + val voxelType = imageRA.get()!!.createVariable() as T + println("addVolume " + image.numDimensions() + " interval " + image as Interval) + + //int numTimepoints = 1; + if (image.numDimensions() > 3) { + timepoints = image.dimension(3).toInt() + } + val ds = RAISource(voxelType, sources, converterSetups, timepoints, cacheControl) + val options = VolumeViewerOptions() + val v: Volume = RAIVolume(ds, options, hub) + v.name = name + v.metadata["sources"] = sources + v.metadata["VoxelDimensions"] = voxelDimensions + v.spatial().scale = Vector3f(1.0f, voxelDimensions[1]/voxelDimensions[0], voxelDimensions[2]/voxelDimensions[0]) * v.pixelToWorldRatio * 10.0f + val tf = v.transferFunction + val rampMin = 0f + val rampMax = 0.1f + tf.clear() + tf.addControlPoint(0.0f, 0.0f) + tf.addControlPoint(rampMin, 0.0f) + tf.addControlPoint(1.0f, rampMax) + val bg = BoundingGrid() + bg.node = v + + imageToVolumeMap[image] = v + return addNode(v, block = block) + } + + /** + * Adds a SourceAndConverter to the scene. + * + * @param sources The list of SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + fun > addVolume(sources: List>, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + var setupId = 0 + val converterSetups = ArrayList() + for (source in sources) { + converterSetups.add(BigDataViewer.createConverterSetup(source, setupId++)) + } + val v = addVolume(sources, converterSetups, numTimepoints, name, *voxelDimensions, block = block) + imageToVolumeMap[sources] = v + return v + } + + /** + * Get the Volume that corresponds to an image if one exists + * @param image an image of any type (e.g. IterableInterval, RAI, SourceAndConverter) + * @return a Volume corresponding to the input image + */ + fun getVolumeFromImage(image: Any): Volume? { + if( image in imageToVolumeMap ) + return imageToVolumeMap[image] + return null + } + + /** + * Update a volume with the given IterableInterval. + * This method actually populates the volume + * @param image image to update into volume + * @param name name of image + * @param voxelDimensions dimensions of voxel in volume + * @param v existing volume to update + * @param pixel type of image + * @return a Node corresponding to the input volume + */ + @Suppress("UNCHECKED_CAST") + fun > updateVolume(image: IterableInterval, name: String, + voxelDimensions: FloatArray, v: Volume): Volume { + val sacs = v.metadata["sources"] as List>? + val source = sacs!![0].spimSource.getSource(0, 0) // hard coded to timepoint and mipmap 0 + val sCur = Views.iterable(source).cursor() + val iCur = image.cursor() + while (sCur.hasNext()) { + sCur.fwd() + iCur.fwd() + sCur.get()!!.set(iCur.get()) + } + v.name = name + v.metadata["VoxelDimensions"] = voxelDimensions + v.volumeManager.notifyUpdate(v) + v.volumeManager.requestRepaint() + //v.getCacheControls().clear(); + //v.setDirty( true ); + v.spatial().needsUpdate = true + //v.setNeedsUpdateWorld( true ); + return v + } + + /** + * + * @return whether PushMode is currently active + */ + fun getPushMode(): Boolean { + return renderer!!.pushMode + } + + /** + * Set the status of PushMode, which only updates the render panel when there is a change in the scene + * @param push true if push mode should be used + * @return current PushMode status + */ + fun setPushMode(push: Boolean): Boolean { + renderer!!.pushMode = push + return renderer!!.pushMode + } + + protected fun finalize() { + stopAnimation() + } + + fun getScenerySettings(): Settings { + return settings + } + + fun getSceneryStatistics(): Statistics { + return stats + } + + fun getSceneryRenderer(): Renderer? { + return renderer + } + + /** + * Enable VR rendering + */ + fun toggleVRRendering() { + vrActive = !vrActive + val cam = scene.activeObserver as? DetachedHeadCamera ?: return + var ti: TrackerInput? = null + var hmdAdded = false + if (!hub.has(SceneryElement.HMDInput)) { + try { + val hmd = OpenVRHMD(false, true) + if (hmd.initializedAndWorking()) { + hub.add(SceneryElement.HMDInput, hmd) + ti = hmd + } else { + logger.warn("Could not initialise VR headset, just activating stereo rendering.") + } + hmdAdded = true + } catch (e: Exception) { + logger.error("Could not add OpenVRHMD: $e") + } + } else { + ti = hub.getWorkingHMD() + } + if (vrActive && ti != null) { + cam.tracker = ti + } else { + cam.tracker = null + } + renderer!!.pushMode = false + // we need to force reloading the renderer as the HMD might require device or instance extensions + if (renderer is VulkanRenderer && hmdAdded) { + replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) + (renderer as VulkanRenderer).toggleVR() + while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { + logger.debug("Waiting for renderer reinitialisation") + try { + Thread.sleep(200) + } catch (e: InterruptedException) { + e.printStackTrace() + } + } + } else { + renderer!!.toggleVR() + } +// renderer!!.toggleVR() + } + + /** + * Set the rotation of Node N by generating a quaternion from the supplied arguments + * @param n node to set rotation for + * @param x x coord of rotation quat + * @param y y coord of rotation quat + * @param z z coord of rotation quat + * @param w w coord of rotation quat + */ + fun setRotation(n: Node, x: Float, y: Float, z: Float, w: Float) { + n.spatialOrNull()?.rotation = Quaternionf(x, y, z, w) + } + + fun setScale(n: Node, x: Float, y: Float, z: Float) { + n.spatialOrNull()?.scale = Vector3f(x, y, z) + } + + @Suppress("UNUSED_PARAMETER") + fun setColor(n: Node, x: Float, y: Float, z: Float, w: Float) { + val col = Vector3f(x, y, z) + n.ifMaterial { + ambient = col + diffuse = col + specular = col + } + } + + fun setPosition(n: Node, x: Float, y: Float, z: Float) { + n.spatialOrNull()?.position = Vector3f(x, y, z) + } + + fun addWindowListener(wl: WindowListener?) { + (mainWindow as? SwingMainWindow)?.addWindowListener(wl) + } + + override fun axis(i: Int): CalibratedAxis? { + return axes[i] + } + + override fun axes(calibratedAxes: Array) { + axes = calibratedAxes + } + + override fun setAxis(calibratedAxis: CalibratedAxis, i: Int) { + axes[i] = calibratedAxis + } + + override fun realMin(i: Int): Double { + return Double.NEGATIVE_INFINITY + } + + override fun realMin(doubles: DoubleArray) { + for (i in doubles.indices) { + doubles[i] = Double.NEGATIVE_INFINITY + } + } + + override fun realMin(realPositionable: RealPositionable) { + for (i in 0 until realPositionable.numDimensions()) { + realPositionable.move(Double.NEGATIVE_INFINITY, i) + } + } + + override fun realMax(i: Int): Double { + return Double.POSITIVE_INFINITY + } + + override fun realMax(doubles: DoubleArray) { + for (i in doubles.indices) { + doubles[i] = Double.POSITIVE_INFINITY + } + } + + override fun realMax(realPositionable: RealPositionable) { + for (i in 0 until realPositionable.numDimensions()) { + realPositionable.move(Double.POSITIVE_INFINITY, i) + } + } + + override fun numDimensions(): Int { + return axes.size + } + + fun setActiveObserver(screenshotCam: Camera?) { + scene.activeObserver = screenshotCam + } + + fun getActiveObserver(): Camera? { + return scene.activeObserver + } + + /** + * Return a list of all nodes that match a given predicate function + * @param nodeMatchPredicate, returns true if a node is a match + * @return list of nodes that match the predicate + */ + fun findNodes(nodeMatchPredicate: Function1): List { + return scene.discover(scene, nodeMatchPredicate, false) + } + + /* + * Convenience function for getting a string of info about a Node + */ + fun nodeInfoString(n: Node): String { + return "Node name: " + n.name + " Node type: " + n.nodeType + " To String: " + n + } + + /** + * Triggers the inspector tree to be completely rebuilt/refreshed. + */ + fun requestPropEditorRefresh() { + eventService.publish(NodeChangedEvent(scene)) + } + + /** + * Triggers the inspector to rebuild/refresh the given node. + * @param n Root of the subtree to get rebuilt/refreshed. + */ + fun requestPropEditorRefresh(n: Node?) { + eventService.publish(NodeChangedEvent(n)) + } + + fun attachCustomPropertyUIToNode(node: Node, ui: CustomPropertyUI) { + node.metadata["sciview-inspector-${ui.module.info.name}"] = ui + } + + fun getAvailableServices() { + println(scijavaContext!!.serviceIndex) + } + + companion object { + //bounds for the controls + const val FPSSPEED_MINBOUND_SLOW = 0.01f + const val FPSSPEED_MAXBOUND_SLOW = 30.0f + const val FPSSPEED_MINBOUND_FAST = 0.2f + const val FPSSPEED_MAXBOUND_FAST = 600f + const val FPSSPEED_MINBOUND_VERYFAST = 10f + const val FPSSPEED_MAXBOUND_VERYFAST = 2000f + + const val MOUSESPEED_MINBOUND = 0.1f + const val MOUSESPEED_MAXBOUND = 3.0f + const val MOUSESCROLL_MINBOUND = 0.3f + const val MOUSESCROLL_MAXBOUND = 10.0f + + @JvmField + val DEFAULT_COLOR: ColorRGB = Colors.LIGHTGRAY + + /** + * Static launching method + * + * @return a newly created SciView + */ + @JvmStatic + @Throws(Exception::class) + fun create(): SciView { + xinitThreads() + val context = Context(ImageJService::class.java, SciJavaService::class.java, SCIFIOService::class.java, + ThreadService::class.java, ObjectService::class.java, LogService::class.java, MenuService::class.java, + IOService::class.java, EventService::class.java, LUTService::class.java, UnitService::class.java, + DatasetIOService::class.java) + val objectService = context.getService(ObjectService::class.java) + objectService.addObject(Utils.SciviewStandalone()) + val sciViewService = context.service(SciViewService::class.java) + return sciViewService.orCreateActiveSciView + } + + /** + * Static launching method + * DEPRECATED use SciView.create() instead + * + * @return a newly created SciView + */ + @Deprecated("Please use SciView.create() instead.", replaceWith = ReplaceWith("SciView.create()")) + @Throws(Exception::class) + fun createSciView(): SciView { + return create() + } + } +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index 7ef1b6d6..a036d1fb 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -1,506 +1,506 @@ -package sc.iview.commands.demo.advanced - -import graphics.scenery.* -import graphics.scenery.attribute.material.Material -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.numerics.Random -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Volume -import org.joml.* -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.log.LogService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class Test: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) - //val calibrationTarget = Icosphere(0.02f, 2) - val TestTarget = Icosphere(0.1f, 2) - - val laser = Cylinder(0.005f, 0.2f, 10) - - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - lateinit var point1:Icosphere - lateinit var point2:Icosphere - - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - val confidenceThreshold = 0.60f - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Backward - @Parameter(label = "Volumes per second") - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - sciview.addChild(TestTarget) - TestTarget.visible = false - - -// sciview.toggleVRRendering() -// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - shell.name = "shell" - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - volume.visible = false - -// point1 = Icosphere(0.1f, 2) -// point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) -// point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} -// sciview.addChild(point1) -// -// point2 = Icosphere(0.1f, 2) -// point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) -// point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} -// sciview.addChild(point2) -// -// var ccc= 0; -// while(true) -// { -// ccc = ccc + 1 -// if(ccc == 1000) -// { -// break -// } -// var w = ccc*0.01f -// point1.position = point1.position.add(Vector3f(w,w,w)) -// point2.position = point2.position.add(Vector3f(w,w,w)) -// val connector = Cylinder(0.1f, 1.0f, 16) -// connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) -//// val connector = Cylinder.betweenPoints(point1.position, point2.position) -// connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} -// sciview.addChild(connector) -// -// } - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addChild(hedgehogs) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - //println("timepoint: "+ newTimepoint); - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - //dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - setupControllerforTracking() - - } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - volume.visible = true - volume.runRecursive { it.visible = true } - playing = true - tracking = true - //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position - - if(true) - { -// val p = Vector3f(0f,0f,-1f) -// referenceTarget.position = p -// referenceTarget.visible = true -// val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) -// val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() -// -// val direction = (pointWorld - headCenter).normalize() - - if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - //System.out.println("tracking!!!!!!!!!!") -// println("direction:"+ direction.toString()) - //addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - showTrack() - } - } - //referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - - - } // bind calibration start to menu key on controller - - } - - private fun showTrack() - { - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-03-10 10.19.49\\Hedgehog_2_2022-03-10 10.21.10.csv") - var volumeDimensions = volume.getDimensions() - var selfdefineworlfmatrix = volume.spatial().world - // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) - selfdefineworlfmatrix = Matrix4f( - 0.015f, 0f, 0f, 0f, - 0f, -0.015f, 0f, 0f, - 0f, 0f, 0.045f, 0f, - -5f, 8f, -2f, 1f - ) - val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix,Vector3f(volumeDimensions)) - print("volume.getDimensions(): "+ volume.getDimensions()) - print("volume.spatial().world: "+ volume.spatial().world) - print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) - - val track = analysis.run() - - print("flag1") - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - print("flag2") - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - - val mInstanced = InstancedNode(master) - sciview.addNode(mInstanced) - - - print("flag3") - if(track == null) - { - return - } - print("flag4") - track.points.windowed(2, 1).forEach { pair -> - - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - - val tp = pair[0].second.timepoint - val pp = Icosphere(0.01f, 1) - pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addNode(pp) - } - } - - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - - - - val p2 = Vector3f(center).add(temp) - - -// print("center position: " + p1.toString()) -// print("p2 position" + p2.toString()) - - TestTarget.visible = true - TestTarget.ifSpatial { position = p2} - - -// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() -// spine.spatial().orientBetweenPoints(p1, p2, true, true) -// spine.visible = true - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) - System.out.println(intersection); - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) -// System.out.println("localEntry:"+ localEntry.toString()) -// System.out.println("localExit:" + localExit.toString()) - - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.2f } - if(count >0 ) - { - println("count of samples: "+ count.toString()) - } - -// spine.metadata["spine"] = metadata -// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ -// fun dumpHedgehog() { -// var lastHedgehog = hedgehogs.children.last() as InstancedNode -// val hedgehogId = hedgehogIds.incrementAndGet() -// -// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() -// val hedgehogFileWriter = hedgehogFile.bufferedWriter() -// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") -// -// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() -// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) -// if(!trackFile.exists()) { -// trackFile.createNewFile() -// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") -// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") -// } -// -// -// val spines = lastHedgehog.instances.mapNotNull { spine -> -// spine.metadata["spine"] as? SpineMetadata -// } -// -// spines.forEach { metadata -> -// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") -// } -// hedgehogFileWriter.close() -// -// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track -// val track = if(existingAnalysis is HedgehogAnalysis.Track) { -// existingAnalysis -// } else { -// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) -// h.run() -// } -// -// if(track == null) { -//// logger.warn("No track returned") -// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) -// return -// } -// -// lastHedgehog.metadata["HedgehogAnalysis"] = track -// lastHedgehog.metadata["Spines"] = spines -// -//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") -// -// val master = if(lastHedgehog == null) { -// val m = Cylinder(3f, 1.0f, 10) -// m.ifMaterial { -// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") -// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) -// roughness = 1.0f -// metallic = 0.0f -// cullingMode = Material.CullingMode.None -// } -// m.name = "Track-$hedgehogId" -// val mInstanced = InstancedNode(m) -// mInstanced -// } else { -// null -// } -// -// val parentId = 0 -// val volumeDimensions = volume.getDimensions() -// -// trackFileWriter.newLine() -// trackFileWriter.newLine() -// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") -// track.points.windowed(2, 1).forEach { pair -> -// if(master != null) { -// val element = master.addInstance() -// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) -// element.parent = volume -// master.instances.add(element) -// } -// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product -// val tp = pair[0].second.timepoint -// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") -// } -// -// master?.let { volume.addChild(it) } -// -// trackFileWriter.close() -// } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.numerics.Random +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.log.LogService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class Test: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + //val calibrationTarget = Icosphere(0.02f, 2) + val TestTarget = Icosphere(0.1f, 2) + + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var point1:Icosphere + lateinit var point2:Icosphere + + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Backward + @Parameter(label = "Volumes per second") + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + sciview.addChild(TestTarget) + TestTarget.visible = false + + +// sciview.toggleVRRendering() +// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + shell.name = "shell" + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + +// point1 = Icosphere(0.1f, 2) +// point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) +// point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} +// sciview.addChild(point1) +// +// point2 = Icosphere(0.1f, 2) +// point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) +// point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} +// sciview.addChild(point2) +// +// var ccc= 0; +// while(true) +// { +// ccc = ccc + 1 +// if(ccc == 1000) +// { +// break +// } +// var w = ccc*0.01f +// point1.position = point1.position.add(Vector3f(w,w,w)) +// point2.position = point2.position.add(Vector3f(w,w,w)) +// val connector = Cylinder(0.1f, 1.0f, 16) +// connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) +//// val connector = Cylinder.betweenPoints(point1.position, point2.position) +// connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} +// sciview.addChild(connector) +// +// } + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + //println("timepoint: "+ newTimepoint); + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + //dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + volume.visible = true + volume.runRecursive { it.visible = true } + playing = false + tracking = true + //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position + + if(true) + { +// val p = Vector3f(0f,0f,-1f) +// referenceTarget.position = p +// referenceTarget.visible = true +// val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) +// val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() +// +// val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + //System.out.println("tracking!!!!!!!!!!") +// println("direction:"+ direction.toString()) + //addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + showTrack() + } + } + //referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + + + } // bind calibration start to menu key on controller + + } + + private fun showTrack() + { + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") + var volumeDimensions = volume.getDimensions() + var selfdefineworlfmatrix = volume.spatial().world + // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) +// selfdefineworlfmatrix = Matrix4f( +// 0.015f, 0f, 0f, 0f, +// 0f, -0.015f, 0f, 0f, +// 0f, 0f, 0.045f, 0f, +// -5f, 8f, -2f, 1f +// ) + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix,Vector3f(volumeDimensions)) + print("volume.getDimensions(): "+ volume.getDimensions()) + print("volume.spatial().world: "+ volume.spatial().world) + print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) + + val track = analysis.run() + + print("flag1") + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + print("flag2") + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + sciview.addNode(mInstanced) + + + print("flag3") + if(track == null) + { + return + } + print("flag4") + track.points.windowed(2, 1).forEach { pair -> + + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + + val tp = pair[0].second.timepoint + val pp = Icosphere(0.01f, 1) + pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addNode(pp) + } + } + + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + + + + val p2 = Vector3f(center).add(temp) + + +// print("center position: " + p1.toString()) +// print("p2 position" + p2.toString()) + + TestTarget.visible = true + TestTarget.ifSpatial { position = p2} + + +// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() +// spine.spatial().orientBetweenPoints(p1, p2, true, true) +// spine.visible = true + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) +// System.out.println("localEntry:"+ localEntry.toString()) +// System.out.println("localExit:" + localExit.toString()) + + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + if(count >0 ) + { + println("count of samples: "+ count.toString()) + } + +// spine.metadata["spine"] = metadata +// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ +// fun dumpHedgehog() { +// var lastHedgehog = hedgehogs.children.last() as InstancedNode +// val hedgehogId = hedgehogIds.incrementAndGet() +// +// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() +// val hedgehogFileWriter = hedgehogFile.bufferedWriter() +// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") +// +// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() +// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) +// if(!trackFile.exists()) { +// trackFile.createNewFile() +// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") +// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") +// } +// +// +// val spines = lastHedgehog.instances.mapNotNull { spine -> +// spine.metadata["spine"] as? SpineMetadata +// } +// +// spines.forEach { metadata -> +// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") +// } +// hedgehogFileWriter.close() +// +// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track +// val track = if(existingAnalysis is HedgehogAnalysis.Track) { +// existingAnalysis +// } else { +// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) +// h.run() +// } +// +// if(track == null) { +//// logger.warn("No track returned") +// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) +// return +// } +// +// lastHedgehog.metadata["HedgehogAnalysis"] = track +// lastHedgehog.metadata["Spines"] = spines +// +//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") +// +// val master = if(lastHedgehog == null) { +// val m = Cylinder(3f, 1.0f, 10) +// m.ifMaterial { +// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") +// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) +// roughness = 1.0f +// metallic = 0.0f +// cullingMode = Material.CullingMode.None +// } +// m.name = "Track-$hedgehogId" +// val mInstanced = InstancedNode(m) +// mInstanced +// } else { +// null +// } +// +// val parentId = 0 +// val volumeDimensions = volume.getDimensions() +// +// trackFileWriter.newLine() +// trackFileWriter.newLine() +// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") +// track.points.windowed(2, 1).forEach { pair -> +// if(master != null) { +// val element = master.addInstance() +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) +// element.parent = volume +// master.instances.add(element) +// } +// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product +// val tp = pair[0].second.timepoint +// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") +// } +// +// master?.let { volume.addChild(it) } +// +// trackFileWriter.close() +// } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 02e3dbe0..1405a21a 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -1,640 +1,636 @@ -package sc.iview.commands.demo.advanced - -import bdv.util.BdvFunctions -import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.* -import org.scijava.Context -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.* -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import org.scijava.ui.behaviour.DragBehaviour -import sc.iview.commands.demo.animation.ParticleDemo -import kotlin.properties.Delegates - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRControllerTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.04f, 2) - val testTarget1 = Icosphere(0.01f, 2) - val testTarget2 = Icosphere(0.04f, 2) - val laser = Cylinder(0.0025f, 1f, 20) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - lateinit var rightController: TrackedDevice - - var hedgehogsList = mutableListOf() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - var rightControllerReady = false - - override fun run() { - - - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) - laser.material().metallic = 0.0f - laser.material().roughness = 1.0f - laser.visible = false - sciview.addNode(laser) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(referenceTarget) - - testTarget1.visible = false - testTarget1.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(testTarget1) - - - testTarget2.visible = false - testTarget2.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(testTarget2) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume -// volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") - if(device.role == TrackerRole.RightHand) { - rightController = device - rightControllerReady = true - log.info("rightController is found and ready") - } -// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) - //dumpHedgehog() - //addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupControllerforTracking() - - } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - println("setupControllerforTracking") - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - tracking = false - dumpHedgehog() - println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) - } else { - addHedgehog() - println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - tracking = true - } - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } - playing = false - - println("test") - - while(true) - { - if(!rightControllerReady) - { - //println("null") - continue - } - else - { - // rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! - if (rightController.model?.spatialOrNull() == null) { - // println("spatial null") - } - else - { - val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() - val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() - - println(headCenter.toString()) - println(pointWorld.toString()) - testTarget1.visible = true - testTarget1.ifSpatial { position = headCenter} - - testTarget2.visible = true - testTarget2.ifSpatial { position = pointWorld} - - laser.visible = true - laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) - - referenceTarget.visible = true - referenceTarget.ifSpatial { position = pointWorld} - - val direction = (pointWorld - headCenter).normalize() - if (tracking) { - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - } - } - - } - - } - - } - - - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) - - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.02f } - //println("cnt: " + count.toString()) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - master.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(master) - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - sciview.addNode(mInstanced) - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - //mInstanced.instances.add(element) - val pp = Icosphere(0.01f, 1) - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import org.scijava.ui.behaviour.DragBehaviour +import sc.iview.commands.demo.animation.ParticleDemo + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRControllerTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.04f, 2) + val testTarget1 = Icosphere(0.01f, 2) + val testTarget2 = Icosphere(0.04f, 2) + val laser = Cylinder(0.0025f, 1f, 20) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var rightController: TrackedDevice + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) + laser.material().metallic = 0.0f + laser.material().roughness = 1.0f + laser.visible = false + sciview.addNode(laser) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(referenceTarget) + + testTarget1.visible = false + testTarget1.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget1) + + + testTarget2.visible = false + testTarget2.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget2) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume +// volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") +// if(device.role == TrackerRole.RightHand) { +// rightController = device +// log.info("rightController is found, its location is in ${rightController.position}") +// } +// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + println("setupControllerforTracking") + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + + println("test") + + while(true) + { + if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) + { + //println("null") + continue + } + else + { + rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! + + if (rightController.model?.spatialOrNull() == null) { + //println("spatial null") + } + else + { + val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() + val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + +// println(headCenter.toString()) +// println(pointWorld.toString()) + testTarget1.visible = true + testTarget1.ifSpatial { position = headCenter} + + testTarget2.visible = true + testTarget2.ifSpatial { position = pointWorld} + + laser.visible = true + laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = pointWorld} + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + + } + + } + + } + + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.02f } + //println("cnt: " + count.toString()) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + //mInstanced.instances.add(element) + val pp = Icosphere(0.01f, 1) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index 71f0a0bf..e45b7f20 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -1,580 +1,583 @@ -package sc.iview.commands.demo.advanced - -import bdv.util.BdvFunctions -import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.* -import org.scijava.Context -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import sc.iview.commands.demo.animation.ParticleDemo - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRHeadSetTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.04f, 2) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - var hedgehogsList = mutableListOf() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume -// volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) - //dumpHedgehog() - //addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupControllerforTracking() - - } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - tracking = false - dumpHedgehog() - println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) - } else { - addHedgehog() - println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - tracking = true - } - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } - playing = false - - - while(true) - { - - val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() - val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() - - referenceTarget.visible = true - referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-2f) } - - val direction = (pointWorld - headCenter).normalize() - if (tracking) { - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - } - } - } - - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) - println("try to find intersection"); - val bbmin = volume.getMaximumBoundingBox().min.xyzw() - val bbmax = volume.getMaximumBoundingBox().max.xyzw() - - val min = volume.spatial().world.transform(bbmin) - val max = volume.spatial().world.transform(bbmax) - println(min) - println(max) - if(intersection is MaybeIntersects.Intersection) { - println("got a intersection") - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.02f } - //println("cnt: " + count.toString()) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - master.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(master) - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - sciview.addNode(mInstanced) - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - //mInstanced.instances.add(element) - val pp = Icosphere(0.01f, 1) - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import sc.iview.commands.demo.animation.ParticleDemo + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRHeadSetTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.04f, 2) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume +// volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = false + + + while(true) + { + + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-2f) } + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + } + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + //println("try to find intersection"); + val bbmin = volume.getMaximumBoundingBox().min.xyzw() + val bbmax = volume.getMaximumBoundingBox().max.xyzw() + + val min = volume.spatial().world.transform(bbmin) + val max = volume.spatial().world.transform(bbmax) + // println(min) + // println(max) + if(intersection is MaybeIntersects.Intersection) { + // println("got a intersection") + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.02f } + //println("cnt: " + count.toString()) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + //mInstanced.instances.add(element) + val tp = pair[0].second.timepoint + val pp = Icosphere(0.01f, 1) + println("trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}") + pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addNode(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file From e1c6013fd599b72472c7be2bc6f0495150acbd70 Mon Sep 17 00:00:00 2001 From: ruoshan Date: Fri, 28 Oct 2022 14:30:19 +0200 Subject: [PATCH 12/55] improve the VRHeadsSettTrackingDemo and fix a bug caused by UV coordinate --- build.gradle.kts | 9 +- settings.gradle.kts | 12 +- .../sc/iview/commands/file/OpenDirofTif.java | 164 +- src/main/kotlin/sc/iview/SciView.kt | 17 +- .../commands/demo/advanced/EyeTrackingDemo.kt | 1346 ++++++++--------- .../demo/advanced/HedgehogAnalysis.kt | 860 +++++------ .../sc/iview/commands/demo/advanced/Test.kt | 100 +- .../demo/advanced/VRControllerTrackingDemo.kt | 6 +- .../demo/advanced/VRHeadSetTrackingDemo.kt | 282 ++-- .../commands/demo/animation/ParticleDemo.vert | 172 +-- 10 files changed, 1511 insertions(+), 1457 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 47207e06..fe20558c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,9 +47,12 @@ dependencies { // if not, uncomment this only to trigger it // api("com.github.scenerygraphics:scenery:$sceneryVersion") - api("org.apache.logging.log4j:log4j-api:2.18.0") - api("org.apache.logging.log4j:log4j-core:2.18.0") - api("org.apache.logging.log4j:log4j-1.2-api:2.18.0") + api("org.apache.logging.log4j:log4j-api:2.19.0") + api("org.apache.logging.log4j:log4j-core:2.19.0") + api("org.apache.logging.log4j:log4j-1.2-api:2.19.0") + + api("org.slf4j:slf4j-api:1.7.36") + api("org.slf4j:slf4j-simple:1.7.36") implementation("com.fasterxml.jackson.core:jackson-databind:2.13.1") implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.1") diff --git a/settings.gradle.kts b/settings.gradle.kts index cc446b56..35428647 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -19,10 +19,10 @@ gradle.rootProject { } val useLocalScenery: String? by extra -if (System.getProperty("CI").toBoolean() != true - && System.getenv("CI").toBoolean() != true - && useLocalScenery?.toBoolean() == true) - if(File("../scenery/build.gradle.kts").exists()) { - logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") +//if (System.getProperty("CI").toBoolean() != true +// && System.getenv("CI").toBoolean() != true +// && useLocalScenery?.toBoolean() == true) +// if(File("../scenery/build.gradle.kts").exists()) { +// logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") includeBuild("../scenery") - } +// } diff --git a/src/main/java/sc/iview/commands/file/OpenDirofTif.java b/src/main/java/sc/iview/commands/file/OpenDirofTif.java index 9c0c0bf6..2b535cd9 100644 --- a/src/main/java/sc/iview/commands/file/OpenDirofTif.java +++ b/src/main/java/sc/iview/commands/file/OpenDirofTif.java @@ -1,79 +1,85 @@ -/*- - * #%L - * Scenery-backed 3D visualization package for ImageJ. - * %% - * Copyright (C) 2016 - 2021 SciView developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package sc.iview.commands.file; - -import org.scijava.command.Command; -import org.scijava.io.IOService; -import org.scijava.log.LogService; -import org.scijava.plugin.Menu; -import org.scijava.plugin.Parameter; -import org.scijava.plugin.Plugin; -import sc.iview.SciView; - -import java.io.File; -import java.io.IOException; - -import static sc.iview.commands.MenuWeights.FILE; -import static sc.iview.commands.MenuWeights.FILE_OPEN; - -/** - * Command to open a file in SciView - * - * @author Kyle Harrington - * - */ -@Plugin(type = Command.class, menuRoot = "SciView", // - menu = { @Menu(label = "File", weight = FILE), // - @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) -public class OpenDirofTif implements Command { - - @Parameter - private IOService io; - - @Parameter - private LogService log; - - @Parameter - private SciView sciView; - - // TODO: Find a more extensible way than hard-coding the extensions. - @Parameter(style = "directory") - private File file; - - @Override - public void run() { - try { - sciView.openDirTiff(file.toPath()); - - } - catch (final IOException | IllegalArgumentException exc) { - log.error( exc ); - } - } -} +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) +public class OpenDirofTif implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "directory") + private File file; + + @Parameter + private int onlyFirst = 0; + + @Override + public void run() { + try { + if(onlyFirst > 0) { + sciView.openDirTiff(file.toPath(), onlyFirst); + } else { + sciView.openDirTiff(file.toPath(), null); + } + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 5c15fa5e..fc8347ee 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -722,9 +722,9 @@ class SciView : SceneryBase, CalibratedRealInterval { } @Throws(IOException::class) - fun openDirTiff(source: Path) + fun openDirTiff(source: Path, onlyFirst: Int? = null) { - val v = Volume.fromPath(source, hub) + val v = Volume.fromPath(source, hub, onlyFirst) v.name = "volume" v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) v.colormap = Colormap.get("jet") @@ -736,12 +736,12 @@ class SciView : SceneryBase, CalibratedRealInterval { v.spatial().wantsComposeModel = true v.spatial().updateWorld(true) - System.out.println("v.model: " + v.model) +// System.out.println("v.model: " + v.model) addChild(v) - System.out.println("v.getDimensions: "+ v.getDimensions()) - - System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) - System.out.println("v.world.matrix: " + v.spatial().world) +// System.out.println("v.getDimensions: "+ v.getDimensions()) +// +// System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) +// System.out.println("v.world.matrix: " + v.spatial().world) } data class PointInTrack( @@ -810,7 +810,7 @@ class SciView : SceneryBase, CalibratedRealInterval { { if(track.trackId > 10) { - // continue + continue } System.out.println("add track: "+ track.trackId.toString() ) val master = Cylinder(0.1f, 1.0f, 10) @@ -840,7 +840,6 @@ class SciView : SceneryBase, CalibratedRealInterval { //mInstanced.instances.add(element) } - } } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 8c3991ed..f632343c 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -1,674 +1,674 @@ -package sc.iview.commands.demo.advanced - -import bdv.util.BdvFunctions -import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew -import graphics.scenery.numerics.OpenSimplexNoise -import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.* -import org.scijava.Context -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class EyeTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) - val calibrationTarget = Icosphere(0.02f, 2) - val laser = Cylinder(0.005f, 0.2f, 10) - - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - val confidenceThreshold = 0.60f - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - calibrationTarget.visible = false - calibrationTarget.material { - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.camera!!.addChild(calibrationTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addChild(hedgehogs) - - val eyeFrames = Mesh("eyeFrames") - val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) - val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) - left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) - left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) - right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) - eyeFrames.addChild(left) - eyeFrames.addChild(right) - - sciview.addChild(eyeFrames) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - pupilTracker.subscribeFrames { eye, texture -> - if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { - return@subscribeFrames - } - - val node = if(eye == 1) { - left - } else { - right - } - - val stream = ByteArrayInputStream(texture) - val image = ImageIO.read(stream) - val data = (image.raster.dataBuffer as DataBufferByte).data - - node.ifMaterial {textures["diffuse"] = Texture( - Vector3i(image.width, image.height, 1), - 3, - UnsignedByteType(), - BufferUtils.allocateByteAndPut(data) - ) } - - lastFrame = System.nanoTime() - } - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogs.visible = false - hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogs.children.removeAt(hedgehogs.children.size-1) - volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> - volume.removeChild(lastTrack) - } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}") - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", duration = 1000) - dumpHedgehog() - addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupCalibration() - - } - - - private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { - val startCalibration = ClickBehaviour { _, _ -> - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - pupilTracker.gazeConfidenceThreshold = confidenceThreshold - if (!pupilTracker.isCalibrated) { - pupilTracker.onCalibrationInProgress = { - cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) - } - - pupilTracker.onCalibrationFailed = { - cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) - } - - pupilTracker.onCalibrationSuccess = { - cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) -// cam.children.find { it.name == "debugBoard" }?.visible = true - - for (i in 0 until 20) { - referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} - Thread.sleep(100) - referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} - Thread.sleep(30) - } - - hmd.removeBehaviour("start_calibration") - hmd.removeKeyBinding("start_calibration") - - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } else { - addHedgehog() - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - } - tracking = !tracking - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } - playing = true - } - - pupilTracker.unsubscribeFrames() - sciview.deleteNode(sciview.find("eyeFrames")) - - log.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) - pupilTracker.calibrate(cam, hmd, - generateReferenceData = true, - calibrationTarget = calibrationTarget) - - pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { - //NEW - PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> - if (gaze.confidence > confidenceThreshold) { - val p = gaze.gazePoint() - referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.position = p - (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" - - val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() - - if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) - } - } - } - -// else -> {gaze-> } - } - - log.info("Calibration routine done.") - } - - // bind calibration start to menu key on controller - - } - } - hmd.addBehaviour("start_calibration", startCalibration) - hmd.addKeyBinding("start_calibration", keybindingCalibration) - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - val spine = (hedgehogs.children.last() as InstancedNode).addInstance() - spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = true - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) -// System.out.println(intersection); - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.2f } - - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - var lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = if(lastHedgehog == null) { - val m = Cylinder(3f, 1.0f, 10) - m.ifMaterial { - ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - roughness = 1.0f - metallic = 0.0f - cullingMode = Material.CullingMode.None - } - m.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(m) - mInstanced - } else { - null - } - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - if(master != null) { - val element = master.addInstance() - element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) - element.parent = volume - master.instances.add(element) - } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - master?.let { volume.addChild(it) } - - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.numerics.OpenSimplexNoise +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class EyeTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.camera!!.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addChild(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.ifMaterial {textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.allocateByteAndPut(data) + ) } + + lastFrame = System.nanoTime() + } + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogs.children.removeAt(hedgehogs.children.size-1) + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}") + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", duration = 1000) + dumpHedgehog() + addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupCalibration() + + } + + + private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { + val startCalibration = ClickBehaviour { _, _ -> + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated) { + pupilTracker.onCalibrationInProgress = { + cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) +// cam.children.find { it.name == "debugBoard" }?.visible = true + + for (i in 0 until 20) { + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} + Thread.sleep(30) + } + + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } else { + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + log.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + //NEW + PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) + } + } + } + +// else -> {gaze-> } + } + + log.info("Calibration routine done.") + } + + // bind calibration start to menu key on controller + + } + } + hmd.addBehaviour("start_calibration", startCalibration) + hmd.addKeyBinding("start_calibration", keybindingCalibration) + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = true + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) +// System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + var lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = if(lastHedgehog == null) { + val m = Cylinder(3f, 1.0f, 10) + m.ifMaterial { + ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + roughness = 1.0f + metallic = 0.0f + cullingMode = Material.CullingMode.None + } + m.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(m) + mInstanced + } else { + null + } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + if(master != null) { + val element = master.addInstance() + element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + element.parent = volume + master.instances.add(element) + } + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + master?.let { volume.addChild(it) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 18e0c3f4..ee3db7dc 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -1,425 +1,435 @@ -package graphics.scenery.bionictracking - -import graphics.scenery.Icosphere -import graphics.scenery.Scene -import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f -import org.joml.Vector3f -import org.joml.Vector4f -import org.joml.Matrix4f -import org.joml.Quaternionf -import graphics.scenery.utils.LazyLogger -import graphics.scenery.utils.extensions.* -import org.slf4j.LoggerFactory -import java.io.File -import kotlin.math.abs -import kotlin.math.pow -import kotlin.math.sqrt - -/** - * - * - * @author Ulrik Günther - */ -class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f, val dimension : Vector3f) { -// val logger by LazyLogger() - - val timepoints = LinkedHashMap>() - - var avgConfidence = 0.0f - private set - var totalSampleCount = 0 - private set - - data class Track( - val points: List>, - val confidence: Float - ) - - init { - logger.info("Starting analysis with ${spines.size} spines") - - spines.forEach { spine -> - val timepoint = spine.timepoint - val current = timepoints[timepoint] - - if(current == null) { - timepoints[timepoint] = arrayListOf(spine) - } else { - current.add(spine) - } - - avgConfidence += spine.confidence - totalSampleCount++ - } - - avgConfidence /= totalSampleCount - } - - private fun localMaxima(list: List): List> = - list.windowed(3, 1).mapIndexed { index, l -> - val left = l[0] - val center = l[1] - val right = l[2] - - // we have a match at center - if(left - center < 0 && center - right > 0) { - index + 1 to center - } else { - null - } - }.filterNotNull() - - data class SpineGraphVertex(val timepoint: Int, - val position: Vector3f, - val worldPosition: Vector3f, - val index: Int, - val value: Float, - val metadata : SpineMetadata, - var previous: SpineGraphVertex? = null, - var next: SpineGraphVertex? = null) { - - fun distance(): Float { - val n = next - return if(n != null) { - val t = (n.worldPosition - this.worldPosition) - sqrt(t.x*t.x + t.y*t.y + t.z*t.z) - } else { - 0.0f - } - } - - fun drop() { - previous?.next = next - next?.previous = previous - } - - override fun toString() : String { - return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" - } - } - - fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) -// fun Iterable.avg() = (this.map { it}.sum() / this.count()) - - fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { - val cross = forward.cross(this) - val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) - - val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) - - return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) - } - - fun run(): Track? { - val startingThreshold = 0.02f - val localMaxThreshold = 0.01f - val zscoreThreshold = 2.0f - val removeTooFarThreshold = 5.0f - - if(timepoints.isEmpty()) { - return null - } - - val startingPoint = timepoints.entries.firstOrNull { entry -> - entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } - } ?: return null - - logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") - -// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) - - timepoints.filter { it.key > startingPoint.key } - .forEach { timepoints.remove(it.key) } - - logger.info("${timepoints.size} timepoints left") - - val candidates = timepoints.map { tp -> - val vs = tp.value.mapIndexedNotNull { i, spine -> - val maxIndices = localMaxima(spine.samples.filterNotNull()) - //logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") - - if(maxIndices.isNotEmpty()) { - - maxIndices.filter { it.first <1200}. - map { index -> -// logger.info(index.toString()) - val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) -// println("i: " + i) -// println("position: " + position) -// println("dimension: "+ dimension) -// println("localToWorld: "+ localToWorld) - val worldPosition = localToWorld.transform((Vector3f(position).mul(dimension)).xyzw()).xyz() -// println("world position: "+ worldPosition) - SpineGraphVertex(tp.key, - position, - worldPosition, - index.first, - index.second, - spine) - - } - } else { - null - } - } - vs - }.flatten() - - - - // get the initial vertex, this one is assumed to always be in front, and have a local max - val initial = candidates.first().filter{it.value>startingThreshold}.first() - System.out.println("initial:"+initial) - System.out.println("candidates number: "+ candidates.size) - var current = initial - var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> -// System.out.println("time: ${time}") -// println("vs: ${vs}") - val distances = vs - .filter { it.value > localMaxThreshold } - .map { vertex -> - val t = current.worldPosition - vertex.worldPosition - val distance = t.length() -// println("current worldposition:"+ current.worldPosition) -// println("vertex.worldposition"+vertex.worldPosition) - vertex to distance - } - .sortedBy { it.second } - //println("distances.size: "+distances.size) - //println("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) -// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) -// { -// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") -// } -// - val closest = distances.firstOrNull()?.first - if(closest != null && distances.firstOrNull()?.second!! >0) { - current.next = closest - closest.previous = current - current = closest - current - } else { - null - } - }.toMutableList() - - - val beforeCount = shortestPath.size - System.out.println("before short path:"+ shortestPath.size) - - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - - fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) - - - while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { - shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - -// println("check which one is removed") -// shortestPath.forEach { -// if(it.distance() >= removeTooFarThreshold * avgPathLength) -// { -// println("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) -// } -// } - } -// - avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - - - var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - while(remaining > 0) { - val outliers = shortestPath - .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - .map { - val idx = shortestPath.indexOf(it) - listOf(idx-1,idx,idx+1) - }.flatten() - - shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - - //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - - remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - //logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - } - - val afterCount = shortestPath.size - logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") -// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") -// logger.info(shortestPath.toString()) - val singlePoints = shortestPath - .groupBy { it.timepoint } - .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } - .filter { - it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f - } - - - logger.info("Returning ${singlePoints.size} points") - - return Track(singlePoints.map { it.position to it}, avgConfidence) - } - - companion object { - private val logger by LazyLogger() - - fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { - logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val confidence = tokens[1].toFloat() - val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - Vector3f(0.0f), - Vector3f(0.0f), - 0.0f, - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Quaternionf(), - Vector3f(0.0f), - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f(), Vector3f()) - } - - private fun String.toVector3f(): Vector3f { -// System.out.println(this) - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - - if (array[0] == "+Inf" || array[0] == "-Inf") - return Vector3f(0.0f,0.0f,0.0f) - - return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) - } - - private fun String.toQuaternionf(): Quaternionf { -// System.out.println(this) - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) - } - fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f, dimension: Vector3f, separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, matrix4f,dimension) - } - - fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f(),Vector3f()) - } - } -} - -fun main(args: Array) { - val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") -// if(args.isEmpty()) { -// logger.error("Sorry, but a file name is needed.") -// return -// } - - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") -// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) - val analysis = HedgehogAnalysis.fromCSV(file) - val results = analysis.run() - logger.info("Results: \n$results") -} +package graphics.scenery.bionictracking + +import graphics.scenery.Icosphere +import graphics.scenery.Scene +import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f +import org.joml.Vector3f +import org.joml.Vector4f +import org.joml.Matrix4f +import org.joml.Quaternionf +import graphics.scenery.utils.LazyLogger +import graphics.scenery.utils.extensions.* +import org.scijava.log.LogService +import org.scijava.plugin.Parameter +import org.slf4j.LoggerFactory +import java.io.File +import kotlin.math.abs +import kotlin.math.pow +import kotlin.math.sqrt + +/** + * + * + * @author Ulrik Günther + */ +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { + + private val logger by LazyLogger() + + val timepoints = LinkedHashMap>() + + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set + + data class Track( + val points: List>, + val confidence: Float + ) + + init { + logger.info("Starting analysis with ${spines.size} spines") + + spines.forEach { spine -> + val timepoint = spine.timepoint + val current = timepoints[timepoint] + + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + private fun localMaxima(list: List): List> = + list.windowed(3, 1).mapIndexed { index, l -> + val left = l[0] + val center = l[1] + val right = l[2] + + // we have a match at center + if(left - center < 0 && center - right > 0) { + index + 1 to center + } else { + null + } + }.filterNotNull() + + data class SpineGraphVertex(val timepoint: Int, + val position: Vector3f, + val worldPosition: Vector3f, + val index: Int, + val value: Float, + val metadata : SpineMetadata, + var previous: SpineGraphVertex? = null, + var next: SpineGraphVertex? = null) { + + fun distance(): Float { + val n = next + return if(n != null) { + val t = (n.worldPosition - this.worldPosition) + sqrt(t.x*t.x + t.y*t.y + t.z*t.z) + } else { + 0.0f + } + } + + fun drop() { + previous?.next = next + next?.previous = previous + } + + override fun toString() : String { + return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" + } + } + + fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) +// fun Iterable.avg() = (this.map { it}.sum() / this.count()) + + fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { + val cross = forward.cross(this) + val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) + + val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) + + return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) + } + + fun run(): Track? { + logger.info("run track analysis") + val startingThreshold = 0.002f + val localMaxThreshold = 0.001f + val zscoreThreshold = 2.0f + val removeTooFarThreshold = 5.0f + + if(timepoints.isEmpty()) { + logger.info("timepoints is empty") + return null + } + + + //step1: find the startingPoint by using startingthreshold + val startingPoint = timepoints.entries.firstOrNull { entry -> + entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } + } ?: return null + + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + +// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) + + timepoints.filter { it.key > startingPoint.key } + .forEach { timepoints.remove(it.key) } + + logger.info("${timepoints.size} timepoints left") + + + //step2: find the maxIndices along the spine + val candidates = timepoints.map { tp -> + val vs = tp.value.mapIndexedNotNull { i, spine -> + val maxIndices = localMaxima(spine.samples.filterNotNull()) +// logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + + if(maxIndices.isNotEmpty()) { +// filter the maxIndices which are too far away + maxIndices.filter { it.first <1200}. + map { index -> +// logger.info(index.toString()) + val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) +// logger.info("i: " + i) +// logger.info("position: " + position) +// logger.info("dimension: "+ dimension) +// logger.info("localToWorld: "+ localToWorld) + val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() +// logger.info("world position: "+ worldPosition) + SpineGraphVertex(tp.key, + position, + worldPosition, + index.first, + index.second, + spine) + + } + } else { + null + } + } + vs + }.flatten() + + + //step3: connect localMaximal points between 2 candidate spines according to the shortest path principle + // get the initial vertex, this one is assumed to always be in front, and have a local max + val initial = candidates.first().filter{it.value>startingThreshold}.first() + logger.info("initial:"+initial) + logger.info("candidates number: "+ candidates.size) + var current = initial + var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> +// System.out.logger.info("time: ${time}") +// logger.info("vs: ${vs}") + val distances = vs + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = t.length() +// logger.info("current worldposition:"+ current.worldPosition) +// logger.info("vertex.worldposition"+vertex.worldPosition) + vertex to distance + } + .sortedBy { it.second } + //logger.info("distances.size: "+distances.size) + //logger.info("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) +// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) +// { +// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") +// } +// + val closest = distances.firstOrNull()?.first + if(closest != null && distances.firstOrNull()?.second!! >0) { + current.next = closest + closest.previous = current + current = closest + current + } else { + null + } + }.toMutableList() + + + val beforeCount = shortestPath.size +// System.out.logger.info("before short path:"+ shortestPath.size) + + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) + + //step4: if some path is longer than multiple average length, it should be removed + while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { + shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + +// logger.info("check which one is removed") +// shortestPath.forEach { +// if(it.distance() >= removeTooFarThreshold * avgPathLength) +// { +// logger.info("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) +// } +// } + } +// + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + + //step5: remove some edges according to zscoreThreshold + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + while(remaining > 0) { + val outliers = shortestPath + .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + .map { + val idx = shortestPath.indexOf(it) + listOf(idx-1,idx,idx+1) + }.flatten() + + shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() + + //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + } + + val afterCount = shortestPath.size + logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") +// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") +// logger.info(shortestPath.toString()) + val singlePoints = shortestPath + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } + .filter { + it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f + } + + + logger.info("Returning ${singlePoints.size} points") + + + return Track(singlePoints.map { it.position to it}, avgConfidence) + } + + companion object { + private val logger by LazyLogger(System.getProperty("scenery.LogLevel", "info")) + + fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { + logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val confidence = tokens[1].toFloat() + val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + Vector3f(0.0f), + Vector3f(0.0f), + 0.0f, + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Quaternionf(), + Vector3f(0.0f), + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + + private fun String.toVector3f(): Vector3f { +// System.out.logger.info(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + + if (array[0] == "+Inf" || array[0] == "-Inf") + return Vector3f(0.0f,0.0f,0.0f) + + return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) + } + + private fun String.toQuaternionf(): Quaternionf { +// System.out.logger.info(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) + } + fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + logger.info("lines number: " + lines.size) + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, matrix4f) + } + + fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + } +} + +fun main(args: Array) { + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") +// if(args.isEmpty()) { +// logger.error("Sorry, but a file name is needed.") +// return +// } + + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") +// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index a036d1fb..dbd20213 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -8,6 +8,7 @@ import graphics.scenery.controls.OpenVRHMD import graphics.scenery.numerics.Random import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.LazyLogger import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus @@ -27,6 +28,7 @@ import java.io.File import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.text.DecimalFormat import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread @@ -39,9 +41,6 @@ class Test: Command{ @Parameter private lateinit var sciview: SciView - @Parameter - private lateinit var log: LogService - lateinit var hmd: OpenVRHMD val referenceTarget = Icosphere(0.004f, 2) //val calibrationTarget = Icosphere(0.02f, 2) @@ -82,6 +81,7 @@ class Test: Command{ var volumeScaleFactor = 1.0f override fun run() { + sciview.addChild(TestTarget) TestTarget.visible = false @@ -114,34 +114,21 @@ class Test: Command{ volume = sciview.find("volume") as Volume volume.visible = false -// point1 = Icosphere(0.1f, 2) -// point1.spatial().position = Vector3f(1.858f,-0.365f,2.432f) -// point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} -// sciview.addChild(point1) -// -// point2 = Icosphere(0.1f, 2) -// point2.spatial().position = Vector3f(1.858f, -0.365f, -10.39f) -// point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} -// sciview.addChild(point2) -// -// var ccc= 0; -// while(true) -// { -// ccc = ccc + 1 -// if(ccc == 1000) -// { -// break -// } -// var w = ccc*0.01f -// point1.position = point1.position.add(Vector3f(w,w,w)) -// point2.position = point2.position.add(Vector3f(w,w,w)) -// val connector = Cylinder(0.1f, 1.0f, 16) -// connector.spatial().orientBetweenPoints(point1.position, point2.position,true,true) -//// val connector = Cylinder.betweenPoints(point1.position, point2.position) -// connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} -// sciview.addChild(connector) -// -// } + point1 = Icosphere(0.1f, 2) + point1.spatial().position = Vector3f(1.858f,2f,8.432f) + point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + sciview.addChild(point1) + + point2 = Icosphere(0.1f, 2) + point2.spatial().position = Vector3f(1.858f, 2f, -10.39f) + point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} + sciview.addChild(point2) + + + val connector = Cylinder.betweenPoints(point1.position, point2.position) + connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(connector) + val bb = BoundingGrid() bb.node = volume @@ -248,7 +235,7 @@ class Test: Command{ val cam = sciview.camera as? DetachedHeadCamera ?: return@thread volume.visible = true volume.runRecursive { it.visible = true } - playing = false + playing = true tracking = true //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position @@ -257,18 +244,19 @@ class Test: Command{ // val p = Vector3f(0f,0f,-1f) // referenceTarget.position = p // referenceTarget.visible = true -// val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) -// val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() + val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() // -// val direction = (pointWorld - headCenter).normalize() + val direction = (pointWorld - headCenter).normalize() if (tracking) { // log.info("Starting spine from $headCenter to $pointWorld") //System.out.println("tracking!!!!!!!!!!") // println("direction:"+ direction.toString()) - //addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) showTrack() } + Thread.sleep(200) } //referenceTarget.visible = true // Pupil has mm units, so we divide by 1000 here to get to scenery units @@ -280,7 +268,9 @@ class Test: Command{ private fun showTrack() { - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") +// val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-10-19 13.48.51\\Hedgehog_1_2022-10-19 13.49.41.csv") + var volumeDimensions = volume.getDimensions() var selfdefineworlfmatrix = volume.spatial().world // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) @@ -290,7 +280,7 @@ class Test: Command{ // 0f, 0f, 0.045f, 0f, // -5f, 8f, -2f, 1f // ) - val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix,Vector3f(volumeDimensions)) + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix) print("volume.getDimensions(): "+ volume.getDimensions()) print("volume.spatial().world: "+ volume.spatial().world) print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) @@ -321,15 +311,19 @@ class Test: Command{ track.points.windowed(2, 1).forEach { pair -> val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0 = Vector3f(pair[0].first)//direct product + val p1 = Vector3f(pair[1].first) val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) val tp = pair[0].second.timepoint - val pp = Icosphere(0.01f, 1) - pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" + val pp = Icosphere(0.1f, 1) + pp.name = "trackpoint_${tp}_${pair[0].first.x}_${pair[0].first.y}_${pair[0].first.z}" +// println("volumeDimensions: " + volumeDimensions) +// println("volume.spatial().world: " + volume.spatial().world) + println("the local position of the point is:" + pair[0].first) + println("the world position of the point is: "+ p0w) pp.spatial().position = p0w pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) sciview.addNode(pp) @@ -348,29 +342,36 @@ class Test: Command{ val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) // print("center position: " + p1.toString()) // print("p2 position" + p2.toString()) - TestTarget.visible = true - TestTarget.ifSpatial { position = p2} +// TestTarget.visible = true +// TestTarget.ifSpatial { position = p2} // val spine = (hedgehogs.children.last() as InstancedNode).addInstance() // spine.spatial().orientBetweenPoints(p1, p2, true, true) // spine.visible = true - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) System.out.println(intersection); if(intersection is MaybeIntersects.Intersection) { // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val nf = DecimalFormat("0.0000") + println("Ray intersects volume at world=${intersection.entry.toString(nf)}/${intersection.exit.toString(nf)} local=${localEntry.toString(nf)}/${localExit.toString(nf)} ") + // System.out.println("localEntry:"+ localEntry.toString()) // System.out.println("localExit:" + localExit.toString()) +// val worldpositionEntry = volume.spatial().world.transform((Vector3f(localEntry)).xyzw()).xyz() +// val worldpositionExit = volume.spatial().world.transform((Vector3f(localExit)).xyzw()).xyz() +// System.out.println("worldEntry:"+ worldpositionEntry.toString()) +// System.out.println("worldExit:" + worldpositionExit.toString()) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null @@ -389,11 +390,10 @@ class Test: Command{ confidence, samples.map { it ?: 0.0f } ) - val count = samples.filterNotNull().count { it > 0.2f } - if(count >0 ) - { + val count = samples.filterNotNull().count { it > 0.002f } + println("count of samples: "+ count.toString()) - } +println(samples) // spine.metadata["spine"] = metadata // spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 1405a21a..9f71631a 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -454,8 +454,8 @@ class VRControllerTrackingDemo: Command{ val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() -// println(headCenter.toString()) -// println(pointWorld.toString()) + println(headCenter.toString()) + println(pointWorld.toString()) testTarget1.visible = true testTarget1.ifSpatial { position = headCenter} @@ -568,7 +568,7 @@ class VRControllerTrackingDemo: Command{ val track = if(existingAnalysis is HedgehogAnalysis.Track) { existingAnalysis } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) h.run() } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index e45b7f20..c7c64690 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -1,47 +1,26 @@ package sc.iview.commands.demo.advanced -import bdv.util.BdvFunctions import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType import graphics.scenery.bionictracking.ConfirmableClickBehaviour import graphics.scenery.bionictracking.HedgehogAnalysis import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* -import org.scijava.Context import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin -import org.scijava.ui.UIService import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte import java.io.BufferedWriter -import java.io.ByteArrayInputStream import java.io.File import java.io.FileWriter import java.nio.file.Files @@ -49,18 +28,19 @@ import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR import org.scijava.log.LogService import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.behaviours.* import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import sc.iview.commands.demo.animation.ParticleDemo +import graphics.scenery.utils.extensions.* +import org.scijava.event.EventService +import sc.iview.commands.file.OpenDirofTif +import sc.iview.event.NodeAddedEvent +import sc.iview.event.NodeChangedEvent +import sc.iview.event.NodeRemovedEvent +import kotlin.concurrent.fixedRateTimer @Plugin(type = Command::class, menuRoot = "SciView", @@ -74,8 +54,11 @@ class VRHeadSetTrackingDemo: Command{ @Parameter private lateinit var log: LogService + @Parameter + private lateinit var eventService: EventService + lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.04f, 2) + val referenceTarget = Icosphere(0.02f, 2) lateinit var sessionId: String lateinit var sessionDirectory: Path @@ -86,6 +69,7 @@ class VRHeadSetTrackingDemo: Command{ var hedgehogVisibility = HedgehogVisibility.Hidden lateinit var volume: Volume + private var selectionStorage: Node? = null enum class PlaybackDirection { Forward, @@ -93,8 +77,8 @@ class VRHeadSetTrackingDemo: Command{ } @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward + var playing = true + var direction = PlaybackDirection.Backward var volumesPerSecond = 1 var skipToNext = false var skipToPrevious = false @@ -127,17 +111,20 @@ class VRHeadSetTrackingDemo: Command{ sciview.addChild(shell) volume = sciview.find("volume") as Volume -// volume.visible = false + +// val testtarget = Icosphere(2f, 2) +// volume.addChild(testtarget) +// testtarget.addAttribute(Grabable::class.java,Grabable()) +// testtarget.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = testtarget})) val bb = BoundingGrid() bb.node = volume bb.visible = false - val debugBoard = TextBoard() debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) debugBoard.text = "" debugBoard.visible = false sciview.camera?.addChild(debugBoard) @@ -196,7 +183,9 @@ class VRHeadSetTrackingDemo: Command{ tracking = false referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() + thread { + dumpHedgehog() + } } } @@ -226,12 +215,14 @@ class VRHeadSetTrackingDemo: Command{ { val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + //LeftController.up.down.left.right sciview.sceneryInputHandler?.let { handler -> hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> + "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), + "move_back" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Down), + "move_left" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Left), + "move_right" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Right) + ).forEach { (name, key) -> handler.getBehaviour(name)?.let { b -> hmd.addBehaviour(name, b) hmd.addKeyBinding(name, key) @@ -280,8 +271,10 @@ class VRHeadSetTrackingDemo: Command{ volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) +// println("volumeScaleFactor is " + volumeScaleFactor) +// println("scale is :" + volume.spatial().scale) } } @@ -290,8 +283,10 @@ class VRHeadSetTrackingDemo: Command{ volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) +// println("volumeScaleFactor is " + volumeScaleFactor) +// println("scale is :" + volume.spatial().scale) } } @@ -304,77 +299,91 @@ class VRHeadSetTrackingDemo: Command{ } } - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - val deleteLastHedgehog = ConfirmableClickBehaviour( armedAction = { timeout -> cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), duration = timeout.toInt()) - }, confirmAction = { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList + if(hedgehogsList.size != 0) + { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList // sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> // sciview.removeChild(lastTrack) // } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), duration = 1000) + } }) - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + val playbackDirection = ClickBehaviour { _, _ -> direction = if(direction == PlaybackDirection.Forward) { PlaybackDirection.Backward } else { PlaybackDirection.Forward } cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) - //dumpHedgehog() - //addHedgehog() } + hmd.addBehaviour("skip_to_next", nextTimepoint) hmd.addBehaviour("skip_to_prev", prevTimepoint) hmd.addBehaviour("faster_or_scale", fasterOrScale) hmd.addBehaviour("slower_or_scale", slowerOrScale) hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("playback_direction",playbackDirection) hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - setupControllerforTracking() + hmd.addKeyBinding("skip_to_next", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Right)) // RightController. right + hmd.addKeyBinding("skip_to_prev", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Left)) // RightController. left + hmd.addKeyBinding("faster_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Up)) // RightController. up + hmd.addKeyBinding("slower_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Down)) //RightController. down + hmd.addKeyBinding("play_pause", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Menu)) // LeftController.Menu + hmd.addKeyBinding("playback_direction", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Menu)) //RightController.Menu + hmd.addKeyBinding("delete_hedgehog", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.A)) //RightController.Side + hmd.addKeyBinding("toggle_hedgehog", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.A)) //LeftController.Side + + + //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) + //left trigger button can validate a track + VRSelect.createAndSet(sciview.currentScene, + hmd, + listOf(OpenVRHMD.OpenVRButton.Trigger), + listOf(TrackerRole.LeftHand), + { n -> + //delete the selected node from volume + println("the spot ${n.name} is selected") + volume.runRecursive{it.removeChild(n)} + eventService.publish(NodeRemovedEvent(n)) + // this is just some action to show a successful selection. + + }, + true) + + VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) + + VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) + setupControllerforTracking() } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { thread { val cam = sciview.camera as? DetachedHeadCamera ?: return@thread @@ -383,8 +392,10 @@ class VRHeadSetTrackingDemo: Command{ referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) tracking = false - dumpHedgehog() - println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + thread { + dumpHedgehog() + println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) + } } else { addHedgehog() println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) @@ -393,13 +404,13 @@ class VRHeadSetTrackingDemo: Command{ tracking = true } } + //RightController.trigger hmd.addBehaviour("toggle_tracking", toggleTracking) hmd.addKeyBinding("toggle_tracking", keybindingTracking) volume.visible = true volume.runRecursive { it.visible = true } - playing = false - +// playing = false while(true) { @@ -408,12 +419,14 @@ class VRHeadSetTrackingDemo: Command{ val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() referenceTarget.visible = true - referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-2f) } + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } val direction = (pointWorld - headCenter).normalize() if (tracking) { addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) } + + Thread.sleep(2) } } @@ -436,19 +449,19 @@ class VRHeadSetTrackingDemo: Command{ val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) //println("try to find intersection"); - val bbmin = volume.getMaximumBoundingBox().min.xyzw() - val bbmax = volume.getMaximumBoundingBox().max.xyzw() - val min = volume.spatial().world.transform(bbmin) - val max = volume.spatial().world.transform(bbmax) - // println(min) - // println(max) if(intersection is MaybeIntersects.Intersection) { // println("got a intersection") // get local entry and exit coordinates, and convert to UV coords + val dim = volume.getDimensions() val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) + val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) + val (samples, localDirection) = volume.sampleRay(entryUV, exitUV) ?: (null to null) + + if (samples != null && localDirection != null) { val metadata = SpineMetadata( @@ -456,8 +469,8 @@ class VRHeadSetTrackingDemo: Command{ center, direction, intersection.distance, - localEntry, - localExit, + entryUV, + exitUV, localDirection, cam.headPosition, cam.headOrientation, @@ -465,8 +478,9 @@ class VRHeadSetTrackingDemo: Command{ confidence, samples.map { it ?: 0.0f } ) - val count = samples.filterNotNull().count { it > 0.02f } - //println("cnt: " + count.toString()) + val count = samples.filterNotNull().count { it > 0.002f } +// println("cnt: " + count.toString()) +// println(samples) spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } // spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } @@ -512,7 +526,8 @@ class VRHeadSetTrackingDemo: Command{ val track = if(existingAnalysis is HedgehogAnalysis.Track) { existingAnalysis } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) + println("do hedgehog Analysis") + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) h.run() } @@ -527,45 +542,57 @@ class VRHeadSetTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - master.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(master) + val parent = RichNode() + parent.name = "Track-$hedgehogId" val parentId = 0 val volumeDimensions = volume.getDimensions() - sciview.addNode(mInstanced) trackFileWriter.newLine() trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - //mInstanced.instances.add(element) - val tp = pair[0].second.timepoint - val pp = Icosphere(0.01f, 1) - println("trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}") - pp.name = "trackpoint_${tp}_${p0w.x}_${p0w.y}_${p0w.z}" - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addNode(pp) + val element = Cylinder(3.0f, 1.0f, 5)//edgeMaster.addInstance() + val p0 = Vector3f(pair[0].first) * Vector3f(volumeDimensions) + val p1 = Vector3f(pair[1].first) * Vector3f(volumeDimensions) + + val tp = pair[0].second.timepoint + + element.spatial().orientBetweenPoints(p0, p1, rescale = true, reposition = true) + element.name = "edge" + element.metadata["Type"] = "edge" + parent.addChild(element) + + val pp = Icosphere(5.0f, 1)//nodeMaster.addInstance() + log.info("Local position: $p0 / $p1") + pp.name = "node-$tp" + pp.metadata["NodeTimepoint"] = tp + pp.metadata["NodePosition"] = p0 + pp.metadata["Type"] = "node" + pp.spatial().position = p0 + + pp.addAttribute(Grabable::class.java, Grabable()) + pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) + pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> + if (device.role == TrackerRole.LeftHand) { + pp.ifSpatial { + position = (device.velocity ?: Vector3f(0.0f)) * 5f + position + eventService.publish(NodeChangedEvent(pp)) + } + } + + })) + parent.addChild(pp) val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } + + + volume.addChild(parent) + eventService.publish(NodeAddedEvent(parent)) + trackFileWriter.close() } @@ -576,8 +603,17 @@ class VRHeadSetTrackingDemo: Command{ fun main(args: Array) { val sv = SciView.create() val command = sv.scijavaContext!!.getService(CommandService::class.java) + + command.run(OpenDirofTif::class.java, true, + hashMapOf( + "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), + "onlyFirst" to 10 + )) + .get() + val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) + command.run(VRHeadSetTrackingDemo::class.java, true, argmap) + .get() } } } \ No newline at end of file diff --git a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert index 4a3d2b12..9d166272 100644 --- a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert +++ b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert @@ -1,86 +1,86 @@ -#version 450 core -#extension GL_ARB_separate_shader_objects: enable - -layout(location = 0) in vec3 vertexPosition; -layout(location = 1) in vec3 vertexNormal; -layout(location = 2) in vec2 vertexTexCoord; -layout(location = 3) in mat4 iModelMatrix; -layout(location = 7) in vec4 vertexColor; - -layout(location = 0) out VertexData { - vec3 FragPosition; - vec3 Normal; - vec2 TexCoord; - vec4 Color; -} Vertex; - -layout(set = 2, binding = 0) uniform Matrices { - mat4 ModelMatrix; - mat4 NormalMatrix; - int isBillboard; -} ubo; - -struct Light { - float Linear; - float Quadratic; - float Intensity; - float Radius; - vec4 Position; - vec4 Color; -}; - -layout(set = 1, binding = 0) uniform LightParameters { - mat4 ViewMatrices[2]; - mat4 InverseViewMatrices[2]; - mat4 ProjectionMatrix; - mat4 InverseProjectionMatrix; - vec3 CamPosition; -}; - -layout(set = 0, binding = 0) uniform VRParameters { - mat4 projectionMatrices[2]; - mat4 inverseProjectionMatrices[2]; - mat4 headShift; - float IPD; - int stereoEnabled; -} vrParameters; - -layout(push_constant) uniform currentEye_t { - int eye; -} currentEye; - -void main() -{ -mat4 mv; - mat4 nMVP; - mat4 projectionMatrix; - - mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] * iModelMatrix + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye] * iModelMatrix); - projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; - - if(ubo.isBillboard > 0) { - mv[0][0] = 1.0f; - mv[0][1] = .0f; - mv[0][2] = .0f; - - mv[1][0] = .0f; - mv[1][1] = 1.0f; - mv[1][2] = .0f; - - mv[2][0] = .0f; - mv[2][1] = .0f; - mv[2][2] = 1.0f; - } - - nMVP = projectionMatrix*mv; - - mat4 normalMatrix = transpose(inverse(iModelMatrix)); - Vertex.Normal = mat3(normalMatrix) * normalize(vertexNormal); - Vertex.TexCoord = vertexTexCoord; - Vertex.FragPosition = vec3(iModelMatrix * vec4(vertexPosition, 1.0)); - Vertex.Color = vertexColor; - - gl_Position = nMVP * vec4(vertexPosition, 1.0); -} - - +#version 450 core +#extension GL_ARB_separate_shader_objects: enable + +layout(location = 0) in vec3 vertexPosition; +layout(location = 1) in vec3 vertexNormal; +layout(location = 2) in vec2 vertexTexCoord; +layout(location = 3) in mat4 iModelMatrix; +layout(location = 7) in vec4 vertexColor; + +layout(location = 0) out VertexData { + vec3 FragPosition; + vec3 Normal; + vec2 TexCoord; + vec4 Color; +} Vertex; + +layout(set = 2, binding = 0) uniform Matrices { + mat4 ModelMatrix; + mat4 NormalMatrix; + int isBillboard; +} ubo; + +struct Light { + float Linear; + float Quadratic; + float Intensity; + float Radius; + vec4 Position; + vec4 Color; +}; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +void main() +{ +mat4 mv; + mat4 nMVP; + mat4 projectionMatrix; + + mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] * iModelMatrix + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye] * iModelMatrix); + projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; + + if(ubo.isBillboard == 1) { + mv[0][0] = 1.0f; + mv[0][1] = .0f; + mv[0][2] = .0f; + + mv[1][0] = .0f; + mv[1][1] = 1.0f; + mv[1][2] = .0f; + + mv[2][0] = .0f; + mv[2][1] = .0f; + mv[2][2] = 1.0f; + } + + nMVP = projectionMatrix*mv; + + mat4 normalMatrix = transpose(inverse(iModelMatrix)); + Vertex.Normal = mat3(normalMatrix) * normalize(vertexNormal); + Vertex.TexCoord = vertexTexCoord; + Vertex.FragPosition = vec3(iModelMatrix * vec4(vertexPosition, 1.0)); + Vertex.Color = vertexColor; + + gl_Position = nMVP * vec4(vertexPosition, 1.0); +} + + From 9348a25ace37ba198d3a3ef5927b64549f9f3d6d Mon Sep 17 00:00:00 2001 From: ruoshan Date: Wed, 9 Nov 2022 11:08:08 +0100 Subject: [PATCH 13/55] add VR interaction to VRHeadSetTrackingDemo and add nodetaggedevent --- build.gradle.kts | 3 +-- settings.gradle.kts | 2 +- src/main/java/sc/iview/event/NodeTaggedEvent.java | 10 ++++++++++ .../demo/advanced/VRHeadSetTrackingDemo.kt | 14 +++++++++----- 4 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 src/main/java/sc/iview/event/NodeTaggedEvent.java diff --git a/build.gradle.kts b/build.gradle.kts index fe20558c..520db877 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -39,8 +39,7 @@ dependencies { } - val sceneryVersion = "4055b8eb32" - //val sceneryVersion = "ce77dda497" + val sceneryVersion = "ccadcc7" api("graphics.scenery:scenery:$sceneryVersion") // check if build is triggered on https://jitpack.io/#scenerygraphics/sciview `build` tab diff --git a/settings.gradle.kts b/settings.gradle.kts index 35428647..c70c6707 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -18,7 +18,7 @@ gradle.rootProject { description = "Scenery-backed 3D visualization package for ImageJ." } -val useLocalScenery: String? by extra +//val useLocalScenery: String? by extra //if (System.getProperty("CI").toBoolean() != true // && System.getenv("CI").toBoolean() != true // && useLocalScenery?.toBoolean() == true) diff --git a/src/main/java/sc/iview/event/NodeTaggedEvent.java b/src/main/java/sc/iview/event/NodeTaggedEvent.java new file mode 100644 index 00000000..23307029 --- /dev/null +++ b/src/main/java/sc/iview/event/NodeTaggedEvent.java @@ -0,0 +1,10 @@ +package sc.iview.event; + +import graphics.scenery.Node; + + +public class NodeTaggedEvent extends NodeEvent { + public NodeTaggedEvent(final Node node ) { + super( node ); + } +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index c7c64690..05543587 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -40,6 +40,7 @@ import sc.iview.commands.file.OpenDirofTif import sc.iview.event.NodeAddedEvent import sc.iview.event.NodeChangedEvent import sc.iview.event.NodeRemovedEvent +import sc.iview.event.NodeTaggedEvent import kotlin.concurrent.fixedRateTimer @Plugin(type = Command::class, @@ -361,21 +362,24 @@ class VRHeadSetTrackingDemo: Command{ //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) - //left trigger button can validate a track + //left trigger button can validate or delete a track VRSelect.createAndSet(sciview.currentScene, hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand), { n -> - //delete the selected node from volume println("the spot ${n.name} is selected") - volume.runRecursive{it.removeChild(n)} - eventService.publish(NodeRemovedEvent(n)) - // this is just some action to show a successful selection. + + //delete the selected node from volume +// volume.runRecursive{it.removeChild(n)} +// eventService.publish(NodeRemovedEvent(n)) + //validate the selected node from volume + eventService.publish(NodeTaggedEvent(n)) }, true) + VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) From 8aa32f4c6a12989a7760ad913739e63b8fe4111a Mon Sep 17 00:00:00 2001 From: ruoshan Date: Wed, 9 Nov 2022 11:26:14 +0100 Subject: [PATCH 14/55] code clean --- settings.gradle.kts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index c70c6707..cc446b56 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -18,11 +18,11 @@ gradle.rootProject { description = "Scenery-backed 3D visualization package for ImageJ." } -//val useLocalScenery: String? by extra -//if (System.getProperty("CI").toBoolean() != true -// && System.getenv("CI").toBoolean() != true -// && useLocalScenery?.toBoolean() == true) -// if(File("../scenery/build.gradle.kts").exists()) { -// logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") +val useLocalScenery: String? by extra +if (System.getProperty("CI").toBoolean() != true + && System.getenv("CI").toBoolean() != true + && useLocalScenery?.toBoolean() == true) + if(File("../scenery/build.gradle.kts").exists()) { + logger.warn("Including local scenery project instead of version declared in build, set -PuseLocalScenery=false to use declared version instead.") includeBuild("../scenery") -// } + } From 1d59a30d169b91874c1295b49c1e60305d36d5db Mon Sep 17 00:00:00 2001 From: ruoshan Date: Tue, 15 Nov 2022 14:58:01 +0100 Subject: [PATCH 15/55] Fix line endings --- build.gradle.kts | 846 ++-- .../openapi/diagnostic/DefaultLogger.java | 172 +- .../sc/iview/commands/file/OpenDirofTif.java | 170 +- .../java/sc/iview/event/NodeTaggedEvent.java | 20 +- src/main/kotlin/sc/iview/SciView.kt | 3772 ++++++++--------- .../commands/demo/advanced/EyeTrackingDemo.kt | 1346 +++--- .../demo/advanced/HedgehogAnalysis.kt | 870 ++-- .../sc/iview/commands/demo/advanced/Test.kt | 1010 ++--- .../demo/advanced/VRControllerTrackingDemo.kt | 1270 +++--- .../demo/advanced/VRHeadSetTrackingDemo.kt | 1244 +++--- .../commands/demo/animation/ParticleDemo.vert | 172 +- 11 files changed, 5446 insertions(+), 5446 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b9399d0b..9f862109 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,423 +1,423 @@ -import org.gradle.kotlin.dsl.implementation -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile -import java.net.URL -import sciview.* - -plugins { - val ktVersion = "1.7.20" - val dokkaVersion = "1.6.21" - - java - kotlin("jvm") version ktVersion - kotlin("kapt") version ktVersion - sciview.publish - sciview.sign - id("org.jetbrains.dokka") version dokkaVersion - jacoco - `maven-publish` - `java-library` - signing -} - -repositories { - mavenCentral() - maven("https://maven.scijava.org/content/groups/public") - maven("https://jitpack.io") -} - -dependencies { - val ktVersion = "1.7.20" - implementation(platform("org.scijava:pom-scijava:31.1.0")) - - // Graphics dependencies - - annotationProcessor("org.scijava:scijava-common:2.90.0") - kapt("org.scijava:scijava-common:2.90.0") { // MANUAL version increment - exclude("org.lwjgl") - } - - val sceneryVersion = "0a32128" - api("graphics.scenery:scenery:$sceneryVersion") { - version { strictly(sceneryVersion) } - exclude("org.biojava.thirdparty", "forester") - exclude("null", "unspecified") - } - - implementation("com.fasterxml.jackson.core:jackson-databind:2.13.4.2") - implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.4") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4") - implementation("org.msgpack:jackson-dataformat-msgpack:0.9.3") - - implementation("com.formdev:flatlaf:2.6") - - // SciJava dependencies - - implementation("org.scijava:scijava-common") - implementation("org.scijava:ui-behaviour") - implementation("org.scijava:script-editor") - implementation("org.scijava:scijava-ui-swing") - implementation("org.scijava:scijava-ui-awt") - implementation("org.scijava:scijava-search") - implementation("org.scijava:scripting-jython") -// implementation(migLayout.swing) - - // ImageJ dependencies - - implementation("net.imagej:imagej-common") - api("net.imagej:imagej-mesh:0.8.1") - implementation("net.imagej:imagej-mesh-io") - implementation("net.imagej:imagej-ops") - implementation("net.imagej:imagej-launcher") - implementation("net.imagej:imagej-ui-swing") - implementation("net.imagej:imagej-legacy") - implementation("io.scif:scifio") - implementation("io.scif:scifio-bf-compat") - - // ImgLib2 dependencies - implementation("net.imglib2:imglib2") - implementation("net.imglib2:imglib2-roi") - - // Math dependencies -// implementation(commons.math3) -// implementation(misc.joml) - - // Kotlin dependencies - implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$ktVersion") - implementation("org.jetbrains.kotlin:kotlin-stdlib:$ktVersion") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") - - // Test scope - -// testImplementation(misc.junit4) - implementation("net.imagej:ij") - implementation("net.imglib2:imglib2-ij") - -// implementation(n5.core) -// implementation(n5.hdf5) -// implementation(n5.imglib2) - implementation("org.janelia.saalfeldlab:n5") - implementation("org.janelia.saalfeldlab:n5-hdf5") - implementation("sc.fiji:spim_data") - - implementation(platform(kotlin("bom"))) - implementation(kotlin("stdlib-jdk8")) - testImplementation(kotlin("test-junit")) - - implementation("sc.fiji:bigdataviewer-core") - implementation("sc.fiji:bigdataviewer-vistools") - - // OME - implementation("ome:formats-bsd") - implementation("ome:formats-gpl") -} - -//kapt { -// useBuildCache = false // safe -// arguments { -// arg("-Werror") -// arg("-Xopt-in", "kotlin.RequiresOptIn") -// } -//} - -tasks { - withType().all { - val version = System.getProperty("java.version").substringBefore('.').toInt() - val default = if (version == 1) "1.8" else "$version" - kotlinOptions { - jvmTarget = project.properties["jvmTarget"]?.toString() ?: default - freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") - } -// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default - } - test { - finalizedBy(jacocoTestReport) // report is always generated after tests run - } - jar { - archiveVersion.set(rootProject.version.toString()) - } - - withType().configureEach { - val matcher = Regex("""generatePomFileFor(\w+)Publication""").matchEntire(name) - val publicationName = matcher?.let { it.groupValues[1] } - - pom.properties.empty() - - pom.withXml { - // Add parent to the generated pom - val parent = asNode().appendNode("parent") - parent.appendNode("groupId", "org.scijava") - parent.appendNode("artifactId", "pom-scijava") - parent.appendNode("version", "31.1.0") - parent.appendNode("relativePath") - - val repositories = asNode().appendNode("repositories") - val jitpackRepo = repositories.appendNode("repository") - jitpackRepo.appendNode("id", "jitpack.io") - jitpackRepo.appendNode("url", "https://jitpack.io") - - val scijavaRepo = repositories.appendNode("repository") - scijavaRepo.appendNode("id", "scijava.public") - scijavaRepo.appendNode("url", "https://maven.scijava.org/content/groups/public") - - // Update the dependencies and properties - val dependenciesNode = asNode().appendNode("dependencies") - val propertiesNode = asNode().appendNode("properties") - propertiesNode.appendNode("inceptionYear", 2016) - - // lwjgl natives - lwjglNatives.forEach { nativePlatform -> - listOf( - "", - "-glfw", - "-jemalloc", - "-opengl", - "-vulkan", - "-openvr", - "-xxhash", - "-remotery", - "-spvc", - "-shaderc", - ).forEach project@ { lwjglProject -> - // Vulkan natives only exist for macOS - if(lwjglProject.endsWith("vulkan") && nativePlatform != "macos") { - return@project - } - - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", "org.lwjgl") - dependencyNode.appendNode("artifactId", "lwjgl$lwjglProject") - dependencyNode.appendNode("version", "\${lwjgl.version}") - dependencyNode.appendNode("classifier", nativePlatform) - dependencyNode.appendNode("scope", "runtime") - } - } - - // jvrpn natives - lwjglNatives.forEach { classifier -> - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", "graphics.scenery") - dependencyNode.appendNode("artifactId", "jvrpn") - dependencyNode.appendNode("version", "\${jvrpn.version}") - dependencyNode.appendNode("classifier", classifier) - dependencyNode.appendNode("scope", "runtime") - } - // add jvrpn property because it only has runtime native deps - propertiesNode.appendNode("jvrpn.version", "1.2.0") - - val versionedArtifacts = listOf("scenery", - "flatlaf", - "kotlin-stdlib-common", - "kotlin-stdlib", - "kotlinx-coroutines-core", - "pom-scijava", - "lwjgl-bom", - "imagej-mesh", - "jackson-module-kotlin", - "jackson-dataformat-yaml", - "jackson-dataformat-msgpack", - "jogl-all", - "kotlin-bom", - "lwjgl", - "lwjgl-glfw", - "lwjgl-jemalloc", - "lwjgl-vulkan", - "lwjgl-opengl", - "lwjgl-openvr", - "lwjgl-xxhash", - "lwjgl-remotery", - "lwjgl-spvc", - "lwjgl-shaderc") - - val toSkip = listOf("pom-scijava") - - configurations.implementation.allDependencies.forEach { - val artifactId = it.name - - if (!toSkip.contains(artifactId)) { - - val propertyName = "$artifactId.version" - - if (versionedArtifacts.contains(artifactId)) { - // add "[version]" to pom - propertiesNode.appendNode(propertyName, it.version) - } - - val dependencyNode = dependenciesNode.appendNode("dependency") - dependencyNode.appendNode("groupId", it.group) - dependencyNode.appendNode("artifactId", artifactId) - dependencyNode.appendNode("version", "\${$propertyName}") - - // Custom per artifact tweaks - println(artifactId) - if ("\\-bom".toRegex().find(artifactId) != null) { - dependencyNode.appendNode("type", "pom") - } - // from https://github.com/scenerygraphics/sciview/pull/399#issuecomment-904732945 - if (artifactId == "formats-gpl") { - val exclusions = dependencyNode.appendNode("exclusions") - val jacksonCore = exclusions.appendNode("exclusion") - jacksonCore.appendNode("groupId", "com.fasterxml.jackson.core") - jacksonCore.appendNode("artifactId", "jackson-core") - val jacksonAnnotations = exclusions.appendNode("exclusion") - jacksonAnnotations.appendNode("groupId", "com.fasterxml.jackson.core") - jacksonAnnotations.appendNode("artifactId", "jackson-annotations") - } - //dependencyNode.appendNode("scope", it.scope) - } - } - - var depStartIdx = "".toRegex().find(asString())?.range?.start - var depEndIdx = "".toRegex().find(asString())?.range?.last - if (depStartIdx != null) { - if (depEndIdx != null) { - asString().replace(depStartIdx, depEndIdx + 1, "") - } - } - - depStartIdx = "".toRegex().find(asString())?.range?.start - depEndIdx = "".toRegex().find(asString())?.range?.last - if (depStartIdx != null) { - if (depEndIdx != null) { - asString().replace(depStartIdx, depEndIdx + 1, "") - } - } - } - } - - dokkaHtml { - enabled = false - dokkaSourceSets.configureEach { - sourceLink { - localDirectory.set(file("src/main/kotlin")) - remoteUrl.set(URL("https://github.com/scenerygraphics/sciview/tree/master/src/main/kotlin")) - remoteLineSuffix.set("#L") - } - } - } - - dokkaJavadoc { - enabled = false - } - - jacocoTestReport { - reports { - xml.required.set(true) - html.required.set(true) - } - dependsOn(test) // tests are required to run before generating the report - } - - register("runMain", JavaExec::class.java) { - classpath = sourceSets.main.get().runtimeClasspath - - main = "sc.iview.Main" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - - register("runImageJMain", JavaExec::class.java) { - classpath = sourceSets.main.get().runtimeClasspath - - main = "sc.iview.ImageJMain" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - - sourceSets.main.get().allSource.files - .filter { it.path.contains("demo") && (it.name.endsWith(".kt") || it.name.endsWith(".java")) } - .map { - val p = it.path - if (p.endsWith(".kt")) { - p.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - } else { - p.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - } - } - .forEach { className -> - println("Working on $className") - val exampleName = className.substringAfterLast(".") - val exampleType = className.substringBeforeLast(".").substringAfterLast(".") - - println("Registering $exampleName of $exampleType") - register(name = className.substringAfterLast(".")) { - classpath = sourceSets.test.get().runtimeClasspath - main = className - group = "demos.$exampleType" - - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - } - } - - register(name = "run") { - classpath = sourceSets.main.get().runtimeClasspath - if (project.hasProperty("target")) { - project.property("target")?.let { target -> - classpath = sourceSets.test.get().runtimeClasspath - - println("Target is $target") - // if(target.endsWith(".kt")) { - // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - // } else { - // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - // } - - main = "$target" - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } - - println("Will run target $target with classpath $classpath, main=$main") - println("JVM arguments passed to target: $allJvmArgs") - } - } - } -} - -val dokkaJavadocJar by tasks.register("dokkaJavadocJar") { - dependsOn(tasks.dokkaJavadoc) - from(tasks.dokkaJavadoc.get().outputDirectory.get()) - archiveClassifier.set("javadoc") -} - -val dokkaHtmlJar by tasks.register("dokkaHtmlJar") { - dependsOn(tasks.dokkaHtml) - from(tasks.dokkaHtml.get().outputDirectory.get()) - archiveClassifier.set("html-doc") -} - -jacoco { - toolVersion = "0.8.7" -} - -artifacts { - archives(dokkaJavadocJar) - archives(dokkaHtmlJar) -} - -java.withSourcesJar() +import org.gradle.kotlin.dsl.implementation +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import java.net.URL +import sciview.* + +plugins { + val ktVersion = "1.7.20" + val dokkaVersion = "1.6.21" + + java + kotlin("jvm") version ktVersion + kotlin("kapt") version ktVersion + sciview.publish + sciview.sign + id("org.jetbrains.dokka") version dokkaVersion + jacoco + `maven-publish` + `java-library` + signing +} + +repositories { + mavenCentral() + maven("https://maven.scijava.org/content/groups/public") + maven("https://jitpack.io") +} + +dependencies { + val ktVersion = "1.7.20" + implementation(platform("org.scijava:pom-scijava:31.1.0")) + + // Graphics dependencies + + annotationProcessor("org.scijava:scijava-common:2.90.0") + kapt("org.scijava:scijava-common:2.90.0") { // MANUAL version increment + exclude("org.lwjgl") + } + + val sceneryVersion = "0a32128" + api("graphics.scenery:scenery:$sceneryVersion") { + version { strictly(sceneryVersion) } + exclude("org.biojava.thirdparty", "forester") + exclude("null", "unspecified") + } + + implementation("com.fasterxml.jackson.core:jackson-databind:2.13.4.2") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.4") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4") + implementation("org.msgpack:jackson-dataformat-msgpack:0.9.3") + + implementation("com.formdev:flatlaf:2.6") + + // SciJava dependencies + + implementation("org.scijava:scijava-common") + implementation("org.scijava:ui-behaviour") + implementation("org.scijava:script-editor") + implementation("org.scijava:scijava-ui-swing") + implementation("org.scijava:scijava-ui-awt") + implementation("org.scijava:scijava-search") + implementation("org.scijava:scripting-jython") +// implementation(migLayout.swing) + + // ImageJ dependencies + + implementation("net.imagej:imagej-common") + api("net.imagej:imagej-mesh:0.8.1") + implementation("net.imagej:imagej-mesh-io") + implementation("net.imagej:imagej-ops") + implementation("net.imagej:imagej-launcher") + implementation("net.imagej:imagej-ui-swing") + implementation("net.imagej:imagej-legacy") + implementation("io.scif:scifio") + implementation("io.scif:scifio-bf-compat") + + // ImgLib2 dependencies + implementation("net.imglib2:imglib2") + implementation("net.imglib2:imglib2-roi") + + // Math dependencies +// implementation(commons.math3) +// implementation(misc.joml) + + // Kotlin dependencies + implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$ktVersion") + implementation("org.jetbrains.kotlin:kotlin-stdlib:$ktVersion") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") + + // Test scope + +// testImplementation(misc.junit4) + implementation("net.imagej:ij") + implementation("net.imglib2:imglib2-ij") + +// implementation(n5.core) +// implementation(n5.hdf5) +// implementation(n5.imglib2) + implementation("org.janelia.saalfeldlab:n5") + implementation("org.janelia.saalfeldlab:n5-hdf5") + implementation("sc.fiji:spim_data") + + implementation(platform(kotlin("bom"))) + implementation(kotlin("stdlib-jdk8")) + testImplementation(kotlin("test-junit")) + + implementation("sc.fiji:bigdataviewer-core") + implementation("sc.fiji:bigdataviewer-vistools") + + // OME + implementation("ome:formats-bsd") + implementation("ome:formats-gpl") +} + +//kapt { +// useBuildCache = false // safe +// arguments { +// arg("-Werror") +// arg("-Xopt-in", "kotlin.RequiresOptIn") +// } +//} + +tasks { + withType().all { + val version = System.getProperty("java.version").substringBefore('.').toInt() + val default = if (version == 1) "1.8" else "$version" + kotlinOptions { + jvmTarget = project.properties["jvmTarget"]?.toString() ?: default + freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") + } +// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default + } + test { + finalizedBy(jacocoTestReport) // report is always generated after tests run + } + jar { + archiveVersion.set(rootProject.version.toString()) + } + + withType().configureEach { + val matcher = Regex("""generatePomFileFor(\w+)Publication""").matchEntire(name) + val publicationName = matcher?.let { it.groupValues[1] } + + pom.properties.empty() + + pom.withXml { + // Add parent to the generated pom + val parent = asNode().appendNode("parent") + parent.appendNode("groupId", "org.scijava") + parent.appendNode("artifactId", "pom-scijava") + parent.appendNode("version", "31.1.0") + parent.appendNode("relativePath") + + val repositories = asNode().appendNode("repositories") + val jitpackRepo = repositories.appendNode("repository") + jitpackRepo.appendNode("id", "jitpack.io") + jitpackRepo.appendNode("url", "https://jitpack.io") + + val scijavaRepo = repositories.appendNode("repository") + scijavaRepo.appendNode("id", "scijava.public") + scijavaRepo.appendNode("url", "https://maven.scijava.org/content/groups/public") + + // Update the dependencies and properties + val dependenciesNode = asNode().appendNode("dependencies") + val propertiesNode = asNode().appendNode("properties") + propertiesNode.appendNode("inceptionYear", 2016) + + // lwjgl natives + lwjglNatives.forEach { nativePlatform -> + listOf( + "", + "-glfw", + "-jemalloc", + "-opengl", + "-vulkan", + "-openvr", + "-xxhash", + "-remotery", + "-spvc", + "-shaderc", + ).forEach project@ { lwjglProject -> + // Vulkan natives only exist for macOS + if(lwjglProject.endsWith("vulkan") && nativePlatform != "macos") { + return@project + } + + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", "org.lwjgl") + dependencyNode.appendNode("artifactId", "lwjgl$lwjglProject") + dependencyNode.appendNode("version", "\${lwjgl.version}") + dependencyNode.appendNode("classifier", nativePlatform) + dependencyNode.appendNode("scope", "runtime") + } + } + + // jvrpn natives + lwjglNatives.forEach { classifier -> + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", "graphics.scenery") + dependencyNode.appendNode("artifactId", "jvrpn") + dependencyNode.appendNode("version", "\${jvrpn.version}") + dependencyNode.appendNode("classifier", classifier) + dependencyNode.appendNode("scope", "runtime") + } + // add jvrpn property because it only has runtime native deps + propertiesNode.appendNode("jvrpn.version", "1.2.0") + + val versionedArtifacts = listOf("scenery", + "flatlaf", + "kotlin-stdlib-common", + "kotlin-stdlib", + "kotlinx-coroutines-core", + "pom-scijava", + "lwjgl-bom", + "imagej-mesh", + "jackson-module-kotlin", + "jackson-dataformat-yaml", + "jackson-dataformat-msgpack", + "jogl-all", + "kotlin-bom", + "lwjgl", + "lwjgl-glfw", + "lwjgl-jemalloc", + "lwjgl-vulkan", + "lwjgl-opengl", + "lwjgl-openvr", + "lwjgl-xxhash", + "lwjgl-remotery", + "lwjgl-spvc", + "lwjgl-shaderc") + + val toSkip = listOf("pom-scijava") + + configurations.implementation.allDependencies.forEach { + val artifactId = it.name + + if (!toSkip.contains(artifactId)) { + + val propertyName = "$artifactId.version" + + if (versionedArtifacts.contains(artifactId)) { + // add "[version]" to pom + propertiesNode.appendNode(propertyName, it.version) + } + + val dependencyNode = dependenciesNode.appendNode("dependency") + dependencyNode.appendNode("groupId", it.group) + dependencyNode.appendNode("artifactId", artifactId) + dependencyNode.appendNode("version", "\${$propertyName}") + + // Custom per artifact tweaks + println(artifactId) + if ("\\-bom".toRegex().find(artifactId) != null) { + dependencyNode.appendNode("type", "pom") + } + // from https://github.com/scenerygraphics/sciview/pull/399#issuecomment-904732945 + if (artifactId == "formats-gpl") { + val exclusions = dependencyNode.appendNode("exclusions") + val jacksonCore = exclusions.appendNode("exclusion") + jacksonCore.appendNode("groupId", "com.fasterxml.jackson.core") + jacksonCore.appendNode("artifactId", "jackson-core") + val jacksonAnnotations = exclusions.appendNode("exclusion") + jacksonAnnotations.appendNode("groupId", "com.fasterxml.jackson.core") + jacksonAnnotations.appendNode("artifactId", "jackson-annotations") + } + //dependencyNode.appendNode("scope", it.scope) + } + } + + var depStartIdx = "".toRegex().find(asString())?.range?.start + var depEndIdx = "".toRegex().find(asString())?.range?.last + if (depStartIdx != null) { + if (depEndIdx != null) { + asString().replace(depStartIdx, depEndIdx + 1, "") + } + } + + depStartIdx = "".toRegex().find(asString())?.range?.start + depEndIdx = "".toRegex().find(asString())?.range?.last + if (depStartIdx != null) { + if (depEndIdx != null) { + asString().replace(depStartIdx, depEndIdx + 1, "") + } + } + } + } + + dokkaHtml { + enabled = false + dokkaSourceSets.configureEach { + sourceLink { + localDirectory.set(file("src/main/kotlin")) + remoteUrl.set(URL("https://github.com/scenerygraphics/sciview/tree/master/src/main/kotlin")) + remoteLineSuffix.set("#L") + } + } + } + + dokkaJavadoc { + enabled = false + } + + jacocoTestReport { + reports { + xml.required.set(true) + html.required.set(true) + } + dependsOn(test) // tests are required to run before generating the report + } + + register("runMain", JavaExec::class.java) { + classpath = sourceSets.main.get().runtimeClasspath + + main = "sc.iview.Main" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + + register("runImageJMain", JavaExec::class.java) { + classpath = sourceSets.main.get().runtimeClasspath + + main = "sc.iview.ImageJMain" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + + sourceSets.main.get().allSource.files + .filter { it.path.contains("demo") && (it.name.endsWith(".kt") || it.name.endsWith(".java")) } + .map { + val p = it.path + if (p.endsWith(".kt")) { + p.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + } else { + p.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + } + } + .forEach { className -> + println("Working on $className") + val exampleName = className.substringAfterLast(".") + val exampleType = className.substringBeforeLast(".").substringAfterLast(".") + + println("Registering $exampleName of $exampleType") + register(name = className.substringAfterLast(".")) { + classpath = sourceSets.test.get().runtimeClasspath + main = className + group = "demos.$exampleType" + + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + } + } + + register(name = "run") { + classpath = sourceSets.main.get().runtimeClasspath + if (project.hasProperty("target")) { + project.property("target")?.let { target -> + classpath = sourceSets.test.get().runtimeClasspath + + println("Target is $target") + // if(target.endsWith(".kt")) { + // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + // } else { + // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + // } + + main = "$target" + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } + + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + } + + println("Will run target $target with classpath $classpath, main=$main") + println("JVM arguments passed to target: $allJvmArgs") + } + } + } +} + +val dokkaJavadocJar by tasks.register("dokkaJavadocJar") { + dependsOn(tasks.dokkaJavadoc) + from(tasks.dokkaJavadoc.get().outputDirectory.get()) + archiveClassifier.set("javadoc") +} + +val dokkaHtmlJar by tasks.register("dokkaHtmlJar") { + dependsOn(tasks.dokkaHtml) + from(tasks.dokkaHtml.get().outputDirectory.get()) + archiveClassifier.set("html-doc") +} + +jacoco { + toolVersion = "0.8.7" +} + +artifacts { + archives(dokkaJavadocJar) + archives(dokkaHtmlJar) +} + +java.withSourcesJar() diff --git a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java index 2af3c655..24119d0e 100644 --- a/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java +++ b/src/main/java/com/intellij/openapi/diagnostic/DefaultLogger.java @@ -1,86 +1,86 @@ -/* - * Copyright 2000-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.openapi.diagnostic; - -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.util.ExceptionUtil; -import org.apache.log4j.Level; -import org.jetbrains.annotations.NonNls; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - - - -public class DefaultLogger extends Logger { - @SuppressWarnings("UnusedParameters") - public DefaultLogger(String category) { } - - @Override - public boolean isDebugEnabled() { - return false; - } - - @Override - public void debug(String message) { } - - @Override - public void debug(Throwable t) { } - - @Override - public void debug(@NonNls String message, Throwable t) { } - - @Override - public void info(String message) { } - - @Override - public void info(String message, Throwable t) { } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void warn(@NonNls String message, @Nullable Throwable t) { - t = checkException(t); - System.err.println("WARN: " + message); - if (t != null) t.printStackTrace(System.err); - } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void error(String message, @Nullable Throwable t, @NotNull String... details) { - t = checkException(t); - message += attachmentsToString(t); - System.err.println("ERROR: " + message); - if (t != null) t.printStackTrace(System.err); - if (details.length > 0) { - System.out.println("details: "); - for (String detail : details) { - System.out.println(detail); - } - } - - AssertionError error = new AssertionError(message); - error.initCause(t); - throw error; - } - - @Override - public void setLevel(Level level) { } - - public static String attachmentsToString(@Nullable Throwable t) { - //noinspection ThrowableResultOfMethodCallIgnored - Throwable rootCause = t == null ? null : ExceptionUtil.getRootCause(t); - return ""; - } -} +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.openapi.diagnostic; + +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.ExceptionUtil; +import org.apache.log4j.Level; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + + +public class DefaultLogger extends Logger { + @SuppressWarnings("UnusedParameters") + public DefaultLogger(String category) { } + + @Override + public boolean isDebugEnabled() { + return false; + } + + @Override + public void debug(String message) { } + + @Override + public void debug(Throwable t) { } + + @Override + public void debug(@NonNls String message, Throwable t) { } + + @Override + public void info(String message) { } + + @Override + public void info(String message, Throwable t) { } + + @Override + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void warn(@NonNls String message, @Nullable Throwable t) { + t = checkException(t); + System.err.println("WARN: " + message); + if (t != null) t.printStackTrace(System.err); + } + + @Override + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void error(String message, @Nullable Throwable t, @NotNull String... details) { + t = checkException(t); + message += attachmentsToString(t); + System.err.println("ERROR: " + message); + if (t != null) t.printStackTrace(System.err); + if (details.length > 0) { + System.out.println("details: "); + for (String detail : details) { + System.out.println(detail); + } + } + + AssertionError error = new AssertionError(message); + error.initCause(t); + throw error; + } + + @Override + public void setLevel(Level level) { } + + public static String attachmentsToString(@Nullable Throwable t) { + //noinspection ThrowableResultOfMethodCallIgnored + Throwable rootCause = t == null ? null : ExceptionUtil.getRootCause(t); + return ""; + } +} diff --git a/src/main/java/sc/iview/commands/file/OpenDirofTif.java b/src/main/java/sc/iview/commands/file/OpenDirofTif.java index 2b535cd9..e565332c 100644 --- a/src/main/java/sc/iview/commands/file/OpenDirofTif.java +++ b/src/main/java/sc/iview/commands/file/OpenDirofTif.java @@ -1,85 +1,85 @@ -/*- - * #%L - * Scenery-backed 3D visualization package for ImageJ. - * %% - * Copyright (C) 2016 - 2021 SciView developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package sc.iview.commands.file; - -import org.scijava.command.Command; -import org.scijava.io.IOService; -import org.scijava.log.LogService; -import org.scijava.plugin.Menu; -import org.scijava.plugin.Parameter; -import org.scijava.plugin.Plugin; -import sc.iview.SciView; - -import java.io.File; -import java.io.IOException; - -import static sc.iview.commands.MenuWeights.FILE; -import static sc.iview.commands.MenuWeights.FILE_OPEN; - -/** - * Command to open a file in SciView - * - * @author Kyle Harrington - * - */ -@Plugin(type = Command.class, menuRoot = "SciView", // - menu = { @Menu(label = "File", weight = FILE), // - @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) -public class OpenDirofTif implements Command { - - @Parameter - private IOService io; - - @Parameter - private LogService log; - - @Parameter - private SciView sciView; - - // TODO: Find a more extensible way than hard-coding the extensions. - @Parameter(style = "directory") - private File file; - - @Parameter - private int onlyFirst = 0; - - @Override - public void run() { - try { - if(onlyFirst > 0) { - sciView.openDirTiff(file.toPath(), onlyFirst); - } else { - sciView.openDirTiff(file.toPath(), null); - } - } - catch (final IOException | IllegalArgumentException exc) { - log.error( exc ); - } - } -} +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview.commands.file; + +import org.scijava.command.Command; +import org.scijava.io.IOService; +import org.scijava.log.LogService; +import org.scijava.plugin.Menu; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import sc.iview.SciView; + +import java.io.File; +import java.io.IOException; + +import static sc.iview.commands.MenuWeights.FILE; +import static sc.iview.commands.MenuWeights.FILE_OPEN; + +/** + * Command to open a file in SciView + * + * @author Kyle Harrington + * + */ +@Plugin(type = Command.class, menuRoot = "SciView", // + menu = { @Menu(label = "File", weight = FILE), // + @Menu(label = "Open Directory of tif...", weight = FILE_OPEN) }) +public class OpenDirofTif implements Command { + + @Parameter + private IOService io; + + @Parameter + private LogService log; + + @Parameter + private SciView sciView; + + // TODO: Find a more extensible way than hard-coding the extensions. + @Parameter(style = "directory") + private File file; + + @Parameter + private int onlyFirst = 0; + + @Override + public void run() { + try { + if(onlyFirst > 0) { + sciView.openDirTiff(file.toPath(), onlyFirst); + } else { + sciView.openDirTiff(file.toPath(), null); + } + } + catch (final IOException | IllegalArgumentException exc) { + log.error( exc ); + } + } +} diff --git a/src/main/java/sc/iview/event/NodeTaggedEvent.java b/src/main/java/sc/iview/event/NodeTaggedEvent.java index 23307029..c0dc6feb 100644 --- a/src/main/java/sc/iview/event/NodeTaggedEvent.java +++ b/src/main/java/sc/iview/event/NodeTaggedEvent.java @@ -1,10 +1,10 @@ -package sc.iview.event; - -import graphics.scenery.Node; - - -public class NodeTaggedEvent extends NodeEvent { - public NodeTaggedEvent(final Node node ) { - super( node ); - } -} +package sc.iview.event; + +import graphics.scenery.Node; + + +public class NodeTaggedEvent extends NodeEvent { + public NodeTaggedEvent(final Node node ) { + super( node ); + } +} diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index fc8347ee..535d5309 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -1,1886 +1,1886 @@ -/*- - * #%L - * Scenery-backed 3D visualization package for ImageJ. - * %% - * Copyright (C) 2016 - 2021 SciView developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package sc.iview - -import bdv.BigDataViewer -import bdv.cache.CacheControl -import bdv.tools.brightness.ConverterSetup -import bdv.util.AxisOrder -import bdv.util.RandomAccessibleIntervalSource -import bdv.util.RandomAccessibleIntervalSource4D -import bdv.util.volatiles.VolatileView -import bdv.viewer.Source -import bdv.viewer.SourceAndConverter -import graphics.scenery.* -import graphics.scenery.Scene.RaycastResult -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.opengl.OpenGLRenderer -import graphics.scenery.backends.vulkan.VulkanRenderer -import graphics.scenery.controls.InputHandler -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackerInput -import graphics.scenery.primitives.* -import graphics.scenery.proteins.Protein -import graphics.scenery.proteins.RibbonDiagram -import graphics.scenery.utils.ExtractsNatives -import graphics.scenery.utils.ExtractsNatives.Companion.getPlatform -import graphics.scenery.utils.LogbackUtils -import graphics.scenery.utils.SceneryPanel -import graphics.scenery.utils.Statistics -import graphics.scenery.utils.extensions.times -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.RAIVolume -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import graphics.scenery.volumes.Volume.Companion.fromXML -import graphics.scenery.volumes.Volume.Companion.setupId -import graphics.scenery.volumes.Volume.VolumeDataSource.RAISource -import io.scif.SCIFIOService -import io.scif.services.DatasetIOService -import net.imagej.Dataset -import net.imagej.ImageJService -import net.imagej.axis.CalibratedAxis -import net.imagej.axis.DefaultAxisType -import net.imagej.axis.DefaultLinearAxis -import net.imagej.interval.CalibratedRealInterval -import net.imagej.lut.LUTService -import net.imagej.mesh.Mesh -import net.imagej.mesh.io.ply.PLYMeshIO -import net.imagej.mesh.io.stl.STLMeshIO -import net.imagej.units.UnitService -import net.imglib2.* -import net.imglib2.display.ColorTable -import net.imglib2.img.Img -import net.imglib2.img.array.ArrayImgs -import net.imglib2.realtransform.AffineTransform3D -import net.imglib2.type.numeric.ARGBType -import net.imglib2.type.numeric.NumericType -import net.imglib2.type.numeric.RealType -import net.imglib2.type.numeric.integer.UnsignedByteType -import net.imglib2.view.Views -import org.joml.Quaternionf -import org.joml.Vector3f -import org.joml.Vector4f -import org.scijava.Context -import org.scijava.`object`.ObjectService -import org.scijava.display.Display -import org.scijava.event.EventHandler -import org.scijava.event.EventService -import org.scijava.io.IOService -import org.scijava.log.LogLevel -import org.scijava.log.LogService -import org.scijava.menu.MenuService -import org.scijava.plugin.Parameter -import org.scijava.service.SciJavaService -import org.scijava.thread.ThreadService -import org.scijava.util.ColorRGB -import org.scijava.util.Colors -import org.scijava.util.VersionUtils -import sc.iview.commands.demo.animation.ParticleDemo -import sc.iview.event.NodeActivatedEvent -import sc.iview.event.NodeAddedEvent -import sc.iview.event.NodeChangedEvent -import sc.iview.event.NodeRemovedEvent -import sc.iview.process.MeshConverter -import sc.iview.ui.CustomPropertyUI -import sc.iview.ui.MainWindow -import sc.iview.ui.SwingMainWindow -import sc.iview.ui.TaskManager -import tpietzsch.example2.VolumeViewerOptions -import java.awt.event.WindowListener -import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.FloatBuffer -import java.nio.file.Path -import java.util.* -import java.util.concurrent.Future -import java.util.function.Consumer -import java.util.function.Function -import java.util.function.Predicate -import java.util.stream.Collectors -import kotlin.collections.ArrayList -import kotlin.collections.HashMap -import kotlin.math.cos -import kotlin.math.sin -import kotlin.system.measureTimeMillis - -/** - * Main SciView class. - * - * @author Kyle Harrington - */ -// we suppress unused warnings here because @Parameter-annotated fields -// get updated automatically by SciJava. -class SciView : SceneryBase, CalibratedRealInterval { - val sceneryPanel = arrayOf(null) - - /* - * Return the default floor object - *//* - * Set the default floor object - */ - /** - * The floor that orients the user in the scene - */ - var floor: Node? = null - protected var vrActive = false - - /** - * The primary camera/observer in the scene - */ - var camera: Camera? = null - set(value) { - field = value - setActiveObserver(field) - } - - lateinit var controls: Controls - val targetArcball: AnimatedCenteringBeforeArcBallControl - get() = controls.targetArcball - - val currentScene: Scene - get() = scene - - /** - * Geometry/Image information of scene - */ - private lateinit var axes: Array - - @Parameter - private lateinit var log: LogService - - @Parameter - private lateinit var menus: MenuService - - @Parameter - private lateinit var io: IOService - - @Parameter - private lateinit var eventService: EventService - - @Parameter - private lateinit var lutService: LUTService - - @Parameter - private lateinit var threadService: ThreadService - - @Parameter - private lateinit var objectService: ObjectService - - @Parameter - private lateinit var unitService: UnitService - - private lateinit var imageToVolumeMap: HashMap - - /** - * Queue keeps track of the currently running animations - */ - private var animations: Queue>? = null - - /** - * Animation pause tracking - */ - private var animating = false - - /** - * This tracks the actively selected Node in the scene - */ - var activeNode: Node? = null - private set - - /* - * Return the SciJava Display that contains SciView - *//* - * Set the SciJava Display - */ var display: Display<*>? = null - - /** - * Return the current SceneryJPanel. This is necessary for custom context menus - * @return panel the current SceneryJPanel - */ - var lights: ArrayList? = null - private set - private val notAbstractNode: Predicate = Predicate { node: Node -> !(node is Camera || node is Light || node === floor) } - var isClosed = false - internal set - - private val notAbstractBranchingFunction = Function { node: Node -> node.children.stream().filter(notAbstractNode).collect(Collectors.toList()) } - - val taskManager = TaskManager() - - // If true, then when a new node is added to the scene, the camera will refocus on this node by default - var centerOnNewNodes = false - - // If true, then when a new node is added the thread will block until the node is added to the scene. This is required for - // centerOnNewNodes - var blockOnNewNodes = false - private var headlight: PointLight? = null - - lateinit var mainWindow: MainWindow - - constructor(context: Context) : super("SciView", 1280, 720, false, context) { - context.inject(this) - } - - constructor(applicationName: String?, windowWidth: Int, windowHeight: Int) : super(applicationName!!, windowWidth, windowHeight, false) - - fun publicGetInputHandler(): InputHandler { - return inputHandler!! - } - - /** - * Toggle video recording with scenery's video recording mechanism - * Note: this video recording may skip frames because it is asynchronous - */ - fun toggleRecordVideo() { - if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie() else (renderer as VulkanRenderer).recordMovie() - } - - /** - * Toggle video recording with scenery's video recording mechanism - * Note: this video recording may skip frames because it is asynchronous - * - * @param filename destination for saving video - * @param overwrite should the file be replaced, otherwise a unique incrementing counter will be appended - */ - fun toggleRecordVideo(filename: String?, overwrite: Boolean) { - if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie(filename!!, overwrite) else (renderer as VulkanRenderer).recordMovie(filename!!, overwrite) - } - - /** - * See [Controls.stashControls]. - */ - fun stashControls() { - controls.stashControls() - } - - /** - * See [Controls.restoreControls] and [Controls.stashControls]. - */ - fun restoreControls() { - controls.restoreControls() - } - - internal fun setRenderer(newRenderer: Renderer) { - renderer = newRenderer - } - - - /** - * Reset the scene to initial conditions - */ - fun reset() { - // Initialize the 3D axes - axes = arrayOf( - DefaultLinearAxis(DefaultAxisType("X", true), "um", 1.0), - DefaultLinearAxis(DefaultAxisType("Y", true), "um", 1.0), - DefaultLinearAxis(DefaultAxisType("Z", true), "um", 1.0) - ) - - // Remove everything except camera - val toRemove = getSceneNodes { n: Node? -> n !is Camera } - for (n in toRemove) { - deleteNode(n, false) - } - - imageToVolumeMap = HashMap() - - // Setup camera - if (camera == null) { - camera = DetachedHeadCamera() - (camera as DetachedHeadCamera).position = Vector3f(0.0f, 1.65f, 0.0f) - scene.addChild(camera as DetachedHeadCamera) - } - camera!!.spatial().position = Vector3f(0.0f, 1.65f, 5.0f) - camera!!.perspectiveCamera(50.0f, windowWidth, windowHeight, 0.1f, 1000.0f) - - // Setup lights - val tetrahedron = arrayOfNulls(4) - tetrahedron[0] = Vector3f(1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[1] = Vector3f(-1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[2] = Vector3f(0.0f, 1.0f, 1.0f / Math.sqrt(2.0).toFloat()) - tetrahedron[3] = Vector3f(0.0f, -1.0f, 1.0f / Math.sqrt(2.0).toFloat()) - lights = ArrayList() - for (i in 0..3) { // TODO allow # initial lights to be customizable? - val light = PointLight(150.0f) - light.spatial().position = tetrahedron[i]!!.mul(25.0f) - light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) - light.intensity = 1.0f - lights!!.add(light) - //camera.addChild( light ); - scene.addChild(light) - } - - // Make a headlight for the camera - headlight = PointLight(150.0f) - headlight!!.spatial().position = Vector3f(0f, 0f, -1f).mul(25.0f) - headlight!!.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) - headlight!!.intensity = 0.5f - headlight!!.name = "headlight" - val lightSphere = Icosphere(1.0f, 2) - headlight!!.addChild(lightSphere) - lightSphere.material().diffuse = headlight!!.emissionColor - lightSphere.material().specular = headlight!!.emissionColor - lightSphere.material().ambient = headlight!!.emissionColor - lightSphere.material().wireframe = true - lightSphere.visible = false - //lights.add( light ); - camera!!.nearPlaneDistance = 0.01f - camera!!.farPlaneDistance = 1000.0f - camera!!.addChild(headlight!!) - floor = InfinitePlane() //new Box( new Vector3f( 500f, 0.2f, 500f ) ); - (floor as InfinitePlane).type = InfinitePlane.Type.Grid - (floor as Node).name = "Floor" - scene.addChild(floor as Node) - } - - /** - * Initialization of SWING and scenery. Also triggers an initial population of lights/camera in the scene - */ - override fun init() { - val logLevel = System.getProperty("scenery.LogLevel", "info") - log.level = LogLevel.value(logLevel) - LogbackUtils.setLogLevel(null, logLevel) - System.getProperties().stringPropertyNames().forEach(Consumer { name: String -> - if (name.startsWith("scenery.LogLevel")) { - LogbackUtils.setLogLevel("", System.getProperty(name, "info")) - } - }) - - // determine imagej-launcher version and to disable Vulkan if XInitThreads() fix - // is not deployed - try { - val launcherClass = Class.forName("net.imagej.launcher.ClassLauncher") - var versionString = VersionUtils.getVersion(launcherClass) - if (versionString != null && getPlatform() == ExtractsNatives.Platform.LINUX) { - versionString = versionString.substring(0, 5) - val launcherVersion = Version(versionString) - val nonWorkingVersion = Version("4.0.5") - if (launcherVersion.compareTo(nonWorkingVersion) <= 0 - && !java.lang.Boolean.parseBoolean(System.getProperty("sciview.DisableLauncherVersionCheck", "false"))) { - logger.info("imagej-launcher version smaller or equal to non-working version ($versionString vs. 4.0.5), disabling Vulkan as rendering backend. Disable check by setting 'scenery.DisableLauncherVersionCheck' system property to 'true'.") - System.setProperty("scenery.Renderer", "OpenGLRenderer") - } else { - logger.info("imagej-launcher version bigger that non-working version ($versionString vs. 4.0.5), all good.") - } - } - } catch (cnfe: ClassNotFoundException) { - // Didn't find the launcher, so we're probably good. - logger.info("imagej-launcher not found, not touching renderer preferences.") - } - - animations = LinkedList() - mainWindow = SwingMainWindow(this) - controls = Controls(this) - - imageToVolumeMap = HashMap() - } - - fun toggleSidebar(): Boolean { - return mainWindow.toggleSidebar() - - } - - private fun initializeInterpreter() { - mainWindow.initializeInterpreter() - } - - /* - * Completely close the SciView window + cleanup - */ - fun closeWindow() { - mainWindow.close() - dispose() - } - - /* - * Return true if the scene has been initialized - */ - val isInitialized: Boolean - get() = sceneInitialized() - - /** - * Place the scene into the center of camera view, and zoom in/out such - * that the whole scene is in the view (everything would be visible if it - * would not be potentially occluded). - */ - fun fitCameraToScene() { - centerOnNode(scene) - //TODO: smooth zoom in/out VLADO vlado Vlado - } - - /** - * Place the scene into the center of camera view. - */ - fun centerOnScene() { - centerOnNode(scene) - } - /* - * Get the InputHandler that is managing mouse, input, VR controls, etc. - */ - val sceneryInputHandler: InputHandler - get() = inputHandler!! - - /* - * Return a bounding box around a subgraph of the scenegraph - */ - fun getSubgraphBoundingBox(n: Node): OrientedBoundingBox? { - val predicate = Function> { node: Node -> node.children } - return getSubgraphBoundingBox(n, predicate) - } - - /* - * Return a bounding box around a subgraph of the scenegraph - */ - fun getSubgraphBoundingBox(n: Node, branchFunction: Function>): OrientedBoundingBox? { - if (n.boundingBox == null && n.children.size != 0) { - return n.getMaximumBoundingBox().asWorld() - } - val branches = branchFunction.apply(n) - if (branches.isEmpty()) { - return if (n.boundingBox == null) null else n.boundingBox!!.asWorld() - } - var bb = n.getMaximumBoundingBox() - for (c in branches) { - val cBB = getSubgraphBoundingBox(c, branchFunction) - if (cBB != null) bb = bb.expand(bb, cBB) - } - return bb - } - - /** - * Place the active node into the center of camera view. - */ - fun centerOnActiveNode() { - if (activeNode == null) return - centerOnNode(activeNode) - } - - /** - * Place the specified node into the center of camera view. - */ - fun centerOnNode(currentNode: Node?) { - if (currentNode == null) { - log.info("Cannot center on null node.") - return - } - - //center the on the same spot as ArcBall does - centerOnPosition(currentNode.getMaximumBoundingBox().getBoundingSphere().origin) - } - - /** - * Center the camera on the specified Node - */ - fun centerOnPosition(currentPos: Vector3f?) { - controls.centerOnPosition(currentPos) - } - - /** - * Activate the node, and center the view on it. - * @param n - * @return the currently active node - */ - fun setActiveCenteredNode(n: Node?): Node? { - //activate... - val ret = setActiveNode(n) - //...and center it - ret?.let { centerOnNode(it) } - return ret - } - - //a couple of shortcut methods to readout controls params - fun getFPSSpeedSlow(): Float { - return controls.getFPSSpeedSlow() - } - - fun getFPSSpeedFast(): Float { - return controls.getFPSSpeedFast() - } - - fun getFPSSpeedVeryFast(): Float { - return controls.getFPSSpeedVeryFast() - } - - fun getMouseSpeed(): Float { - return controls.getMouseSpeed() - } - - fun getMouseScrollSpeed(): Float { - return controls.getMouseScrollSpeed() - } - - //a couple of setters with scene sensible boundary checks - fun setFPSSpeedSlow(slowSpeed: Float) { - controls.setFPSSpeedSlow(slowSpeed) - } - - fun setFPSSpeedFast(fastSpeed: Float) { - controls.setFPSSpeedFast(fastSpeed) - } - - fun setFPSSpeedVeryFast(veryFastSpeed: Float) { - controls.setFPSSpeedVeryFast(veryFastSpeed) - } - - fun setFPSSpeed(newBaseSpeed: Float) { - controls.setFPSSpeed(newBaseSpeed) - } - - fun setMouseSpeed(newSpeed: Float) { - controls.setMouseSpeed(newSpeed) - } - - fun setMouseScrollSpeed(newSpeed: Float) { - controls.setMouseScrollSpeed(newSpeed) - } - - fun setObjectSelectionMode() { - controls.setObjectSelectionMode() - } - - /* - * Set the action used during object selection - */ - fun setObjectSelectionMode(selectAction: Function3?) { - controls.setObjectSelectionMode(selectAction) - } - - fun showContextNodeChooser(x: Int, y: Int) { - mainWindow.showContextNodeChooser(x,y) - } - - /* - * Initial configuration of the scenery InputHandler - * This is automatically called and should not be used directly - */ - override fun inputSetup() { - log.info("Running InputSetup") - controls.inputSetup() - } - - /** - * Add a box at the specified position with specified size, color, and normals on the inside/outside - * @param position position to put the box - * @param size size of the box - * @param color color of the box - * @param inside are normals inside the box? - * @return the Node corresponding to the box - */ - @JvmOverloads - fun addBox(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), size: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, - inside: Boolean = false, block: Box.() -> Unit = {}): Box { - val box = Box(size, inside) - box.spatial().position = position - box.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - return addNode(box, block = block) - } - - /** - * Add a unit sphere at a given [position] with given [radius] and [color]. - * @return the Node corresponding to the sphere - */ - @JvmOverloads - fun addSphere(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), radius: Float = 1f, color: ColorRGB = DEFAULT_COLOR, block: Sphere.() -> Unit = {}): Sphere { - val sphere = Sphere(radius, 20) - sphere.spatial().position = position - sphere.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - - return addNode(sphere, block = block) - } - - /** - * Add a Cylinder at the given position with radius, height, and number of faces/segments - * @param position position of the cylinder - * @param radius radius of the cylinder - * @param height height of the cylinder - * @param num_segments number of segments to represent the cylinder - * @return the Node corresponding to the cylinder - */ - fun addCylinder(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cylinder.() -> Unit = {}): Cylinder { - val cyl = Cylinder(radius, height, num_segments) - cyl.spatial().position = position - return addNode(cyl, block = block) - } - - /** - * Add a Cone at the given position with radius, height, and number of faces/segments - * @param position position to put the cone - * @param radius radius of the cone - * @param height height of the cone - * @param num_segments number of segments used to represent cone - * @return the Node corresponding to the cone - */ - fun addCone(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cone.() -> Unit = {}): Cone { - val cone = Cone(radius, height, num_segments, Vector3f(0.0f, 0.0f, 1.0f)) - cone.spatial().position = position - return addNode(cone, block = block) - } - - /** - * Add a line from start to stop with the given color - * @param start start position of line - * @param stop stop position of line - * @param color color of line - * @return the Node corresponding to the line - */ - @JvmOverloads - fun addLine(start: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), stop: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, block: Line.() -> Unit = {}): Line { - return addLine(arrayOf(start, stop), color, 0.1, block) - } - - /** - * Add a multi-segment line that goes through the supplied points with a single color and edge width - * @param points points along line including first and terminal points - * @param color color of line - * @param edgeWidth width of line segments - * @return the Node corresponding to the line - */ - @JvmOverloads - fun addLine(points: Array, color: ColorRGB, edgeWidth: Double, block: Line.() -> Unit = {}): Line { - val line = Line(points.size) - for (pt in points) { - line.addPoint(pt) - } - line.edgeWidth = edgeWidth.toFloat() - line.material { - ambient = Vector3f(1.0f, 1.0f, 1.0f) - diffuse = Utils.convertToVector3f(color) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - line.spatial().position = points[0] - return addNode(line, block = block) - } - - /** - * Add a PointLight source at the origin - * @return a Node corresponding to the PointLight - */ - @JvmOverloads - fun addPointLight(block: PointLight.() -> Unit = {}): PointLight { - val light = PointLight(5.0f) - light.material { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Vector3f(0.0f, 1.0f, 0.0f) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - light.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - lights!!.add(light) - return addNode(light, block = block) - } - - /** - * Position all lights that were initialized by default around the scene in a circle at Y=0 - */ - fun surroundLighting() { - val bb = getSubgraphBoundingBox(scene, notAbstractBranchingFunction) - val (c, r) = bb!!.getBoundingSphere() - // Choose a good y-position, then place lights around the cross-section through this plane - val y = 0f - for (k in lights!!.indices) { - val light = lights!![k] - val x = (c.x() + r * cos(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() - val z = (c.y() + r * sin(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() - light.lightRadius = 2 * r - light.spatial().position = Vector3f(x, y, z) - } - } - - @Throws(IOException::class) - fun openDirTiff(source: Path, onlyFirst: Int? = null) - { - val v = Volume.fromPath(source, hub, onlyFirst) - v.name = "volume" - v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) - v.colormap = Colormap.get("jet") - v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) - v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) - v.metadata["animating"] = true - v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) - v.visible = true - - v.spatial().wantsComposeModel = true - v.spatial().updateWorld(true) -// System.out.println("v.model: " + v.model) - addChild(v) -// System.out.println("v.getDimensions: "+ v.getDimensions()) -// -// System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) -// System.out.println("v.world.matrix: " + v.spatial().world) - } - - data class PointInTrack( - val t: Int, - val loc: Vector3f, - val cellId: Long, - val parentId: Long, - val nodeScore: Float, - val edgeScore: Float - ) - - data class Track( - val track: List, - val trackId: Int - ) - - @Throws(IOException::class) - fun openTrackFile(file: File) - { - val lines = file.readLines() - var track = ArrayList() - val tracks = ArrayList() - val separator = "," - - var lastTrackId = -1 - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val t = tokens[0].toInt() - val z = tokens[1].toFloat() -2000f - val y = tokens[2].toFloat() -800f - val x = tokens[3].toFloat() -1300f - val cellId = tokens[4].toLong() - val parentId = tokens[5].toLong() - val trackId = tokens[6].toInt() - val nodeScore = tokens[7].toFloat() - val edgeScore = tokens[8].toFloat()/45.0f - - val currentPointInTrack = PointInTrack( - t, - Vector3f(x,y,z), - cellId, - parentId, - nodeScore, - edgeScore - ) - if(lastTrackId != trackId) - { - lastTrackId = trackId - val sortedTrack = track.sortedBy { it.t } - tracks.add(Track(sortedTrack, trackId)) - - track.clear() - } - track.add(currentPointInTrack) - } - val timeCost = measureTimeMillis { - addTracks(tracks) - } - println("time: $timeCost") - } - - fun addTracks(tracks: ArrayList) - { - val rng = Random(17) - for(track in tracks) - { - if(track.trackId > 10) - { - continue - } - System.out.println("add track: "+ track.trackId.toString() ) - val master = Cylinder(0.1f, 1.0f, 10) -// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Vector3f(0.05f, 0f, 0f) - metallic = 0.01f - roughness = 0.5f - } - - val mInstanced = InstancedNode(master) - mInstanced.name = "TrackID-${track.trackId}" - mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } - addNode(mInstanced) - - var cnt = 0 - val a = rng.nextFloat() - val b = rng.nextFloat() - track.track.windowed(2,1).forEach { pair -> - cnt = cnt + 1 - val element = mInstanced.addInstance() - element.name ="EdgeID-$cnt" - element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} - element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) - //mInstanced.instances.add(element) - - } - } - - } - - - /** - * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud - * @param source string of a data source - * @throws IOException - */ - @Suppress("UNCHECKED_CAST") - @Throws(IOException::class) - fun open(source: String) { - if (source.endsWith(".xml", ignoreCase = true)) { - addNode(fromXML(source, hub, VolumeViewerOptions())) - return - } else if (source.takeLast(4).equals(".pdb", true)) { - val protein = Protein.fromFile(source) - val ribbon = RibbonDiagram(protein) - ribbon.spatial().position = Vector3f(0f, 0f, 0f) - addNode(ribbon) - return - } else if (source.endsWith(".stl", ignoreCase = true)) { - val stlReader = STLMeshIO() - addMesh(stlReader.open(source)) - return - } else if (source.endsWith(".ply", ignoreCase = true)) { - val plyReader = PLYMeshIO() - addMesh(plyReader.open(source)) - return - } - val data = io.open(source) - if (data is Mesh) - addMesh(data) - else if (data is graphics.scenery.Mesh) - addMesh(data) - else if (data is PointCloud) - addPointCloud(data) - else if (data is Dataset) - addVolume(data) - else if (data is RandomAccessibleInterval<*>) - addVolume(data as RandomAccessibleInterval>, source) - else if (data is List<*>) { - val list = data - require(!list.isEmpty()) { "Data source '$source' appears empty." } - val element = list[0] - if (element is RealLocalizable) { - // NB: For now, we assume all elements will be RealLocalizable. - // Highly likely to be the case, barring antagonistic importers. - val points = list as List - addPointCloud(points, source) - } else { - val type = if (element == null) "" else element.javaClass.name - throw IllegalArgumentException("Data source '" + source + // - "' contains elements of unknown type '" + type + "'") - } - } else { - val type = if (data == null) "" else data.javaClass.name - throw IllegalArgumentException("Data source '" + source + // - "' contains data of unknown type '" + type + "'") - } - } - - /** - * Add the given points to the scene as a PointCloud with a given name - * @param points points to use in a PointCloud - * @param name name of the PointCloud - * @return - */ - @JvmOverloads - fun addPointCloud(points: Collection, - name: String? = "PointCloud", - pointSize : Float = 1.0f, - block: PointCloud.() -> Unit = {}): PointCloud { - val flatVerts = FloatArray(points.size * 3) - var k = 0 - for (point in points) { - flatVerts[k * 3] = point.getFloatPosition(0) - flatVerts[k * 3 + 1] = point.getFloatPosition(1) - flatVerts[k * 3 + 2] = point.getFloatPosition(2) - k++ - } - val pointCloud = PointCloud(pointSize, name!!) - val vBuffer: FloatBuffer = BufferUtils.allocateFloat(flatVerts.size * 4) - val nBuffer: FloatBuffer = BufferUtils.allocateFloat(0) - vBuffer.put(flatVerts) - vBuffer.flip() - pointCloud.geometry().vertices = vBuffer - pointCloud.geometry().normals = nBuffer - pointCloud.geometry().indices = BufferUtils.allocateInt(0) - pointCloud.spatial().position = Vector3f(0f, 0f, 0f) - - pointCloud.setupPointCloud() - return addNode(pointCloud, block = block) - } - - /** - * Add a PointCloud to the scene - * @param pointCloud existing PointCloud to add to scene - * @return a Node corresponding to the PointCloud - */ - @JvmOverloads - fun addPointCloud(pointCloud: PointCloud, block: PointCloud.() -> Unit = {}): PointCloud { - pointCloud.setupPointCloud() - pointCloud.spatial().position = Vector3f(0f, 0f, 0f) - return addNode(pointCloud, block = block) - } - - /** - * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true - * @param n node to add to scene - * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services - * @return a Node corresponding to the Node - */ - @JvmOverloads - fun addNode(n: N, activePublish: Boolean = true, block: N.() -> Unit = {}): N { - n?.let { - it.block() - scene.addChild(it) - objectService.addObject(n) - if (blockOnNewNodes) { - Utils.blockWhile({ this.find(n.name) == null }, 20) - //System.out.println("find(name) " + find(n.getName()) ); - } - // Set new node as active and centered? - setActiveNode(n) - if (centerOnNewNodes) { - centerOnNode(n) - } - if (activePublish) { - eventService.publish(NodeAddedEvent(n)) - } - } - return n - } - - /** - * Add a scenery Mesh to the scene - * @param scMesh scenery mesh to add to scene - * @return a Node corresponding to the mesh - */ - fun addMesh(scMesh: graphics.scenery.Mesh): graphics.scenery.Mesh { - scMesh.ifMaterial { - ambient = Vector3f(1.0f, 0.0f, 0.0f) - diffuse = Vector3f(0.0f, 1.0f, 0.0f) - specular = Vector3f(1.0f, 1.0f, 1.0f) - } - scMesh.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - objectService.addObject(scMesh) - return addNode(scMesh) - } - - /** - * Add an ImageJ mesh to the scene - * @param mesh net.imagej.mesh to add to scene - * @return a Node corresponding to the mesh - */ - fun addMesh(mesh: Mesh): graphics.scenery.Mesh { - val scMesh = MeshConverter.toScenery(mesh) - return addMesh(scMesh) - } - - /** - * [Deprecated: use deleteNode] - * Remove a Mesh from the scene - * @param scMesh mesh to remove from scene - */ - fun removeMesh(scMesh: graphics.scenery.Mesh?) { - scene.removeChild(scMesh!!) - } - - /** - * Set the currently active node - * @param n existing node that should become active focus of this SciView - * @return the currently active node - */ - fun setActiveNode(n: Node?): Node? { - if (activeNode === n) return activeNode - activeNode = n - targetArcball.target = { n?.getMaximumBoundingBox()?.getBoundingSphere()?.origin ?: Vector3f(0.0f, 0.0f, 0.0f) } - mainWindow.selectNode(activeNode) - eventService.publish(NodeActivatedEvent(activeNode)) - return activeNode - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeAdded(event: NodeAddedEvent?) { - mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeRemoved(event: NodeRemovedEvent?) { - mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeChanged(event: NodeChangedEvent?) { - // TODO: Check if rebuilding the tree is necessary here, otherwise this costs a lot of performance - //mainWindow.rebuildSceneTree() - } - - @Suppress("UNUSED_PARAMETER") - @EventHandler - protected fun onNodeActivated(event: NodeActivatedEvent?) { - // TODO: add listener code for node activation, if necessary - // NOTE: do not update property window here, this will lead to a loop. - } - - fun toggleInspectorWindow() { - toggleSidebar() - } - - @Suppress("UNUSED_PARAMETER") - fun setInspectorWindowVisibility(visible: Boolean) { -// inspector.setVisible(visible); -// if( visible ) -// mainSplitPane.setDividerLocation(getWindowWidth()/4 * 3); -// else -// mainSplitPane.setDividerLocation(getWindowWidth()); - } - - @Suppress("UNUSED_PARAMETER") - fun setInterpreterWindowVisibility(visible: Boolean) { -// interpreterPane.getComponent().setVisible(visible); -// if( visible ) -// interpreterSplitPane.setDividerLocation(getWindowHeight()/10 * 6); -// else -// interpreterSplitPane.setDividerLocation(getWindowHeight()); - } - - /** - * Create an animation thread with the given fps speed and the specified action - * @param fps frames per second at which this action should be run - * @param action Runnable that contains code to run fps times per second - * @return a Future corresponding to the thread - */ - @Synchronized - fun animate(fps: Int, action: Runnable): Future<*> { - // TODO: Make animation speed less laggy and more accurate. - val delay = 1000 / fps - val thread = threadService.run { - while (animating) { - action.run() - try { - Thread.sleep(delay.toLong()) - } catch (e: InterruptedException) { - break - } - } - } - animations!!.add(thread) - animating = true - return thread - } - - /** - * Stop all animations - */ - @Synchronized - fun stopAnimation() { - animating = false - while (!animations!!.isEmpty()) { - animations!!.peek().cancel(true) - animations!!.remove() - } - } - - /** - * Take a screenshot and save it to the default scenery location - */ - fun takeScreenshot() { - renderer!!.screenshot() - } - - /** - * Take a screenshot and save it to the specified path - * @param path path for saving the screenshot - */ - fun takeScreenshot(path: String?, overwrite: Boolean = false) { - renderer!!.screenshot(path!!, overwrite = overwrite) - } - - /** - * Take a screenshot and return it as an Img - * @return an Img of type UnsignedByteType - */ - val screenshot: Img - get() { - val screenshot = getSceneryRenderer()?.requestScreenshot() ?: throw IllegalStateException("No renderer present, cannot create screenshot") - return ArrayImgs.unsignedBytes(screenshot.data!!, screenshot.width.toLong(), screenshot.height.toLong(), 4L) - } - - /** - * Take a screenshot and return it as an Img - * @return an Img of type UnsignedByteType - */ - val aRGBScreenshot: Img - get() { - return Utils.convertToARGB(screenshot) - } - - /** - * @param name The name of the node to find. - * @return the node object or null, if the node has not been found. - */ - fun find(name: String): Node? { - val n = scene.find(name) - if (n == null) { - logger.warn("Node with name $name not found.") - } - return n - } - - /** - * @return an array of all nodes in the scene except Cameras and PointLights - */ - val sceneNodes: Array - get() = getSceneNodes { n: Node? -> n !is Camera && n !is PointLight } - - /** - * Get a list of nodes filtered by filter predicate - * @param filter, a predicate that filters the candidate nodes - * @return all nodes that match the predicate - */ - fun getSceneNodes(filter: Predicate): Array { - return scene.children.filter{ filter.test(it) }.toTypedArray() - } - - /** - * @return an array of all Node's in the scene - */ - val allSceneNodes: Array - get() = getSceneNodes { _: Node? -> true } - - /** - * Delete the current active node - */ - fun deleteActiveNode() { - deleteNode(activeNode) - } - - /** - * Delete a specified node and control whether the event is published - * @param node node to delete from scene - * @param activePublish whether the deletion should be published - */ - @JvmOverloads - fun deleteNode(node: Node?, activePublish: Boolean = true) { - if(node is Volume) { - node.volumeManager.remove(node) - val toRemove = ArrayList() - for( entry in imageToVolumeMap.entries ) { - if( entry.value == node ) { - toRemove.add(entry.key) - } - } - for(entry in toRemove) { - imageToVolumeMap.remove(entry) - } - } - - for (child in node!!.children) { - deleteNode(child, activePublish) - } - objectService.removeObject(node) - node.parent?.removeChild(node) - if (activeNode == node) { - setActiveNode(null) - } - //maintain consistency - if( activePublish ) { - eventService.publish(NodeRemovedEvent(node)) - } - } - - /** - * Dispose the current scenery renderer, hub, and other scenery things - */ - fun dispose() { - val objs: List = objectService.getObjects(Node::class.java) - for (obj in objs) { - objectService.removeObject(obj) - } - scijavaContext!!.service(SciViewService::class.java).close(this) - close() - // if scijavaContext was not created by ImageJ, then system exit - if( objectService.getObjects(Utils.SciviewStandalone::class.java).size > 0 ) { - log.info("Was running as sciview standalone, shutting down JVM") - System.exit(0) - } - } - - override fun close() { - super.close() - } - - /** - * Move the current active camera to the specified position - * @param position position to move the camera to - */ - fun moveCamera(position: FloatArray) { - camera?.spatial()?.position = Vector3f(position[0], position[1], position[2]) - } - - /** - * Move the current active camera to the specified position - * @param position position to move the camera to - */ - fun moveCamera(position: DoubleArray) { - camera?.spatial()?.position = Vector3f(position[0].toFloat(), position[1].toFloat(), position[2].toFloat()) - } - - /** - * Get the current application name - * @return a String of the application name - */ - fun getName(): String { - return applicationName - } - - /** - * Add a child to the scene. you probably want addNode - * @param node node to add as a child to the scene - */ - fun addChild(node: Node) { - scene.addChild(node) - } - - /** - * Add a Dataset to the scene as a volume. Voxel resolution and name are extracted from the Dataset itself - * @param image image to add as a volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - fun addVolume(image: Dataset, block: Volume.() -> Unit = {}): Volume { - val voxelDims = FloatArray(image.numDimensions()) - - for (d in voxelDims.indices) { - val inValue = image.axis(d).averageScale(0.0, 1.0) - if (image.axis(d).unit() == null) { - voxelDims[d] = inValue.toFloat() - } else { - val imageAxisUnit = image.axis(d).unit().replace("µ", "u") - val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") - - voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() - } - } - - logger.info("Adding with ${voxelDims.joinToString(",")}") - val v = addVolume(image, voxelDims, block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a Dataset as a Volume with the specified voxel dimensions - * @param image image to add as a volume - * @param voxelDimensions dimensions of voxels in volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - @Suppress("UNCHECKED_CAST") - fun addVolume(image: Dataset, voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image.imgPlus as RandomAccessibleInterval>, image.name ?: "Volume", - *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image, name, 1f, 1f, 1f, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - fun > addVolume(image: RandomAccessibleInterval, voxelDimensions: FloatArray, block: Volume.() -> Unit): Volume { - val v = addVolume(image, "volume", *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add an IterableInterval as a Volume - * @param image - * @param - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, "Volume") - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - - /** - * Add an IterableInterval as a Volume - * @param image image to add as a volume - * @param name name of image - * @param pixel type of image - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval, name: String = "Volume"): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, name, 1f, 1f, 1f) - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - - /** - * Set the colormap using an ImageJ LUT name - * @param n node to apply colormap to - * @param lutName name of LUT according to imagej LUTService - */ - fun setColormap(n: Node, lutName: String) { - try { - setColormap(n, lutService.loadLUT(lutService.findLUTs()[lutName])) - } catch (e: IOException) { - e.printStackTrace() - } - } - - /** - * Set the ColorMap of node n to the supplied colorTable - * @param n node to apply colortable to - * @param colorTable ColorTable to use - */ - fun setColormap(n: Node, colorTable: ColorTable) { - val copies = 16 - val byteBuffer = ByteBuffer.allocateDirect( - 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture - val tmp = ByteArray(4 * colorTable.length) - for (k in 0 until colorTable.length) { - for (c in 0 until colorTable.componentCount) { - // TODO this assumes numBits is 8, could be 16 - tmp[4 * k + c] = colorTable[c, k].toByte() - } - if (colorTable.componentCount == 3) { - tmp[4 * k + 3] = 255.toByte() - } - } - for (i in 0 until copies) { - byteBuffer.put(tmp) - } - byteBuffer.flip() - n.metadata["sciviewColormap"] = colorTable - if (n is Volume) { - n.colormap = Colormap.fromColorTable(colorTable) - n.geometryOrNull()?.dirty = true - n.spatial().needsUpdate = true - } - } - - /** - * Adss a SourceAndConverter to the scene. - * - * @param sac The SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - fun > addVolume(sac: SourceAndConverter, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - val sources: MutableList> = ArrayList() - sources.add(sac) - - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) - imageToVolumeMap[sources] = v - imageToVolumeMap[sac] = v - return v - } - - /** - * Add an IterableInterval to the image with the specified voxelDimensions and name - * This version of addVolume does most of the work - * @param image image to add as a volume - * @param name name of image - * @param voxelDimensions dimensions of voxel in volume - * @param pixel type of image - * @return a Node corresponding to the Volume - */ - @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", - vararg voxelDimensions: Float, block: Volume.() -> Unit = {}): Volume { - //log.debug( "Add Volume " + name + " image: " + image ); - val dimensions = LongArray(image.numDimensions()) - image.dimensions(dimensions) - val minPt = LongArray(image.numDimensions()) - - // Get type at min point - val imageRA = image.randomAccess() - image.min(minPt) - imageRA.setPosition(minPt) - val voxelType = imageRA.get()!!.createVariable() - val converterSetups: ArrayList = ArrayList() - val stacks = AxisOrder.splitInputStackIntoSourceStacks(image, AxisOrder.getAxisOrder(AxisOrder.DEFAULT, image, false)) - val sourceTransform = AffineTransform3D() - val sources: ArrayList> = ArrayList>() - var numTimepoints = 1 - for (stack in stacks) { - var s: Source - if (stack.numDimensions() > 3) { - numTimepoints = (stack.max(3) + 1).toInt() - s = RandomAccessibleIntervalSource4D(stack, voxelType, sourceTransform, name) - } else { - s = RandomAccessibleIntervalSource(stack, voxelType, sourceTransform, name) - } - val source = BigDataViewer.wrapWithTransformedSource( - SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) - converterSetups.add(BigDataViewer.createConverterSetup(source, setupId.getAndIncrement())) - sources.add(source) - } - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) - v.metadata.set("RandomAccessibleInterval", image) - imageToVolumeMap[image] = v - return v - } - - /** - * Adds a SourceAndConverter to the scene. - * - * This method actually instantiates the volume. - * - * @param sources The list of SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - @Suppress("UNCHECKED_CAST") - fun > addVolume(sources: List>, - converterSetups: ArrayList, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - var timepoints = numTimepoints - var cacheControl: CacheControl? = null - -// RandomAccessibleInterval image = -// ((RandomAccessibleIntervalSource4D) sources.get(0).getSpimSource()). -// .getSource(0, 0); - val image = sources[0].spimSource.getSource(0, 0) - if (image is VolatileView<*, *>) { - val viewData = (image as VolatileView>).volatileViewData - cacheControl = viewData.cacheControl - } - val dimensions = LongArray(image.numDimensions()) - image.dimensions(dimensions) - val minPt = LongArray(image.numDimensions()) - - // Get type at min point - val imageRA = image.randomAccess() - image.min(minPt) - imageRA.setPosition(minPt) - val voxelType = imageRA.get()!!.createVariable() as T - println("addVolume " + image.numDimensions() + " interval " + image as Interval) - - //int numTimepoints = 1; - if (image.numDimensions() > 3) { - timepoints = image.dimension(3).toInt() - } - val ds = RAISource(voxelType, sources, converterSetups, timepoints, cacheControl) - val options = VolumeViewerOptions() - val v: Volume = RAIVolume(ds, options, hub) - v.name = name - v.metadata["sources"] = sources - v.metadata["VoxelDimensions"] = voxelDimensions - v.spatial().scale = Vector3f(1.0f, voxelDimensions[1]/voxelDimensions[0], voxelDimensions[2]/voxelDimensions[0]) * v.pixelToWorldRatio * 10.0f - val tf = v.transferFunction - val rampMin = 0f - val rampMax = 0.1f - tf.clear() - tf.addControlPoint(0.0f, 0.0f) - tf.addControlPoint(rampMin, 0.0f) - tf.addControlPoint(1.0f, rampMax) - val bg = BoundingGrid() - bg.node = v - - imageToVolumeMap[image] = v - return addNode(v, block = block) - } - - /** - * Adds a SourceAndConverter to the scene. - * - * @param sources The list of SourceAndConverter to add - * @param name Name of the dataset - * @param voxelDimensions Array with voxel dimensions. - * @param Type of the dataset. - * @return THe node corresponding to the volume just added. - */ - @JvmOverloads - fun > addVolume(sources: List>, - numTimepoints: Int, - name: String = "Volume", - vararg voxelDimensions: Float, - block: Volume.() -> Unit = {}): Volume { - var setupId = 0 - val converterSetups = ArrayList() - for (source in sources) { - converterSetups.add(BigDataViewer.createConverterSetup(source, setupId++)) - } - val v = addVolume(sources, converterSetups, numTimepoints, name, *voxelDimensions, block = block) - imageToVolumeMap[sources] = v - return v - } - - /** - * Get the Volume that corresponds to an image if one exists - * @param image an image of any type (e.g. IterableInterval, RAI, SourceAndConverter) - * @return a Volume corresponding to the input image - */ - fun getVolumeFromImage(image: Any): Volume? { - if( image in imageToVolumeMap ) - return imageToVolumeMap[image] - return null - } - - /** - * Update a volume with the given IterableInterval. - * This method actually populates the volume - * @param image image to update into volume - * @param name name of image - * @param voxelDimensions dimensions of voxel in volume - * @param v existing volume to update - * @param pixel type of image - * @return a Node corresponding to the input volume - */ - @Suppress("UNCHECKED_CAST") - fun > updateVolume(image: IterableInterval, name: String, - voxelDimensions: FloatArray, v: Volume): Volume { - val sacs = v.metadata["sources"] as List>? - val source = sacs!![0].spimSource.getSource(0, 0) // hard coded to timepoint and mipmap 0 - val sCur = Views.iterable(source).cursor() - val iCur = image.cursor() - while (sCur.hasNext()) { - sCur.fwd() - iCur.fwd() - sCur.get()!!.set(iCur.get()) - } - v.name = name - v.metadata["VoxelDimensions"] = voxelDimensions - v.volumeManager.notifyUpdate(v) - v.volumeManager.requestRepaint() - //v.getCacheControls().clear(); - //v.setDirty( true ); - v.spatial().needsUpdate = true - //v.setNeedsUpdateWorld( true ); - return v - } - - /** - * - * @return whether PushMode is currently active - */ - fun getPushMode(): Boolean { - return renderer!!.pushMode - } - - /** - * Set the status of PushMode, which only updates the render panel when there is a change in the scene - * @param push true if push mode should be used - * @return current PushMode status - */ - fun setPushMode(push: Boolean): Boolean { - renderer!!.pushMode = push - return renderer!!.pushMode - } - - protected fun finalize() { - stopAnimation() - } - - fun getScenerySettings(): Settings { - return settings - } - - fun getSceneryStatistics(): Statistics { - return stats - } - - fun getSceneryRenderer(): Renderer? { - return renderer - } - - /** - * Enable VR rendering - */ - fun toggleVRRendering() { - vrActive = !vrActive - val cam = scene.activeObserver as? DetachedHeadCamera ?: return - var ti: TrackerInput? = null - var hmdAdded = false - if (!hub.has(SceneryElement.HMDInput)) { - try { - val hmd = OpenVRHMD(false, true) - if (hmd.initializedAndWorking()) { - hub.add(SceneryElement.HMDInput, hmd) - ti = hmd - } else { - logger.warn("Could not initialise VR headset, just activating stereo rendering.") - } - hmdAdded = true - } catch (e: Exception) { - logger.error("Could not add OpenVRHMD: $e") - } - } else { - ti = hub.getWorkingHMD() - } - if (vrActive && ti != null) { - cam.tracker = ti - } else { - cam.tracker = null - } - renderer!!.pushMode = false - // we need to force reloading the renderer as the HMD might require device or instance extensions - if (renderer is VulkanRenderer && hmdAdded) { - replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) - (renderer as VulkanRenderer).toggleVR() - while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { - logger.debug("Waiting for renderer reinitialisation") - try { - Thread.sleep(200) - } catch (e: InterruptedException) { - e.printStackTrace() - } - } - } else { - renderer!!.toggleVR() - } -// renderer!!.toggleVR() - } - - /** - * Set the rotation of Node N by generating a quaternion from the supplied arguments - * @param n node to set rotation for - * @param x x coord of rotation quat - * @param y y coord of rotation quat - * @param z z coord of rotation quat - * @param w w coord of rotation quat - */ - fun setRotation(n: Node, x: Float, y: Float, z: Float, w: Float) { - n.spatialOrNull()?.rotation = Quaternionf(x, y, z, w) - } - - fun setScale(n: Node, x: Float, y: Float, z: Float) { - n.spatialOrNull()?.scale = Vector3f(x, y, z) - } - - @Suppress("UNUSED_PARAMETER") - fun setColor(n: Node, x: Float, y: Float, z: Float, w: Float) { - val col = Vector3f(x, y, z) - n.ifMaterial { - ambient = col - diffuse = col - specular = col - } - } - - fun setPosition(n: Node, x: Float, y: Float, z: Float) { - n.spatialOrNull()?.position = Vector3f(x, y, z) - } - - fun addWindowListener(wl: WindowListener?) { - (mainWindow as? SwingMainWindow)?.addWindowListener(wl) - } - - override fun axis(i: Int): CalibratedAxis? { - return axes[i] - } - - override fun axes(calibratedAxes: Array) { - axes = calibratedAxes - } - - override fun setAxis(calibratedAxis: CalibratedAxis, i: Int) { - axes[i] = calibratedAxis - } - - override fun realMin(i: Int): Double { - return Double.NEGATIVE_INFINITY - } - - override fun realMin(doubles: DoubleArray) { - for (i in doubles.indices) { - doubles[i] = Double.NEGATIVE_INFINITY - } - } - - override fun realMin(realPositionable: RealPositionable) { - for (i in 0 until realPositionable.numDimensions()) { - realPositionable.move(Double.NEGATIVE_INFINITY, i) - } - } - - override fun realMax(i: Int): Double { - return Double.POSITIVE_INFINITY - } - - override fun realMax(doubles: DoubleArray) { - for (i in doubles.indices) { - doubles[i] = Double.POSITIVE_INFINITY - } - } - - override fun realMax(realPositionable: RealPositionable) { - for (i in 0 until realPositionable.numDimensions()) { - realPositionable.move(Double.POSITIVE_INFINITY, i) - } - } - - override fun numDimensions(): Int { - return axes.size - } - - fun setActiveObserver(screenshotCam: Camera?) { - scene.activeObserver = screenshotCam - } - - fun getActiveObserver(): Camera? { - return scene.activeObserver - } - - /** - * Return a list of all nodes that match a given predicate function - * @param nodeMatchPredicate, returns true if a node is a match - * @return list of nodes that match the predicate - */ - fun findNodes(nodeMatchPredicate: Function1): List { - return scene.discover(scene, nodeMatchPredicate, false) - } - - /* - * Convenience function for getting a string of info about a Node - */ - fun nodeInfoString(n: Node): String { - return "Node name: " + n.name + " Node type: " + n.nodeType + " To String: " + n - } - - /** - * Triggers the inspector tree to be completely rebuilt/refreshed. - */ - fun requestPropEditorRefresh() { - eventService.publish(NodeChangedEvent(scene)) - } - - /** - * Triggers the inspector to rebuild/refresh the given node. - * @param n Root of the subtree to get rebuilt/refreshed. - */ - fun requestPropEditorRefresh(n: Node?) { - eventService.publish(NodeChangedEvent(n)) - } - - fun attachCustomPropertyUIToNode(node: Node, ui: CustomPropertyUI) { - node.metadata["sciview-inspector-${ui.module.info.name}"] = ui - } - - fun getAvailableServices() { - println(scijavaContext!!.serviceIndex) - } - - companion object { - //bounds for the controls - const val FPSSPEED_MINBOUND_SLOW = 0.01f - const val FPSSPEED_MAXBOUND_SLOW = 30.0f - const val FPSSPEED_MINBOUND_FAST = 0.2f - const val FPSSPEED_MAXBOUND_FAST = 600f - const val FPSSPEED_MINBOUND_VERYFAST = 10f - const val FPSSPEED_MAXBOUND_VERYFAST = 2000f - - const val MOUSESPEED_MINBOUND = 0.1f - const val MOUSESPEED_MAXBOUND = 3.0f - const val MOUSESCROLL_MINBOUND = 0.3f - const val MOUSESCROLL_MAXBOUND = 10.0f - - @JvmField - val DEFAULT_COLOR: ColorRGB = Colors.LIGHTGRAY - - /** - * Static launching method - * - * @return a newly created SciView - */ - @JvmStatic - @Throws(Exception::class) - fun create(): SciView { - xinitThreads() - val context = Context(ImageJService::class.java, SciJavaService::class.java, SCIFIOService::class.java, - ThreadService::class.java, ObjectService::class.java, LogService::class.java, MenuService::class.java, - IOService::class.java, EventService::class.java, LUTService::class.java, UnitService::class.java, - DatasetIOService::class.java) - val objectService = context.getService(ObjectService::class.java) - objectService.addObject(Utils.SciviewStandalone()) - val sciViewService = context.service(SciViewService::class.java) - return sciViewService.orCreateActiveSciView - } - - /** - * Static launching method - * DEPRECATED use SciView.create() instead - * - * @return a newly created SciView - */ - @Deprecated("Please use SciView.create() instead.", replaceWith = ReplaceWith("SciView.create()")) - @Throws(Exception::class) - fun createSciView(): SciView { - return create() - } - } -} +/*- + * #%L + * Scenery-backed 3D visualization package for ImageJ. + * %% + * Copyright (C) 2016 - 2021 SciView developers. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package sc.iview + +import bdv.BigDataViewer +import bdv.cache.CacheControl +import bdv.tools.brightness.ConverterSetup +import bdv.util.AxisOrder +import bdv.util.RandomAccessibleIntervalSource +import bdv.util.RandomAccessibleIntervalSource4D +import bdv.util.volatiles.VolatileView +import bdv.viewer.Source +import bdv.viewer.SourceAndConverter +import graphics.scenery.* +import graphics.scenery.Scene.RaycastResult +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.opengl.OpenGLRenderer +import graphics.scenery.backends.vulkan.VulkanRenderer +import graphics.scenery.controls.InputHandler +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerInput +import graphics.scenery.primitives.* +import graphics.scenery.proteins.Protein +import graphics.scenery.proteins.RibbonDiagram +import graphics.scenery.utils.ExtractsNatives +import graphics.scenery.utils.ExtractsNatives.Companion.getPlatform +import graphics.scenery.utils.LogbackUtils +import graphics.scenery.utils.SceneryPanel +import graphics.scenery.utils.Statistics +import graphics.scenery.utils.extensions.times +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.Volume.Companion.fromXML +import graphics.scenery.volumes.Volume.Companion.setupId +import graphics.scenery.volumes.Volume.VolumeDataSource.RAISource +import io.scif.SCIFIOService +import io.scif.services.DatasetIOService +import net.imagej.Dataset +import net.imagej.ImageJService +import net.imagej.axis.CalibratedAxis +import net.imagej.axis.DefaultAxisType +import net.imagej.axis.DefaultLinearAxis +import net.imagej.interval.CalibratedRealInterval +import net.imagej.lut.LUTService +import net.imagej.mesh.Mesh +import net.imagej.mesh.io.ply.PLYMeshIO +import net.imagej.mesh.io.stl.STLMeshIO +import net.imagej.units.UnitService +import net.imglib2.* +import net.imglib2.display.ColorTable +import net.imglib2.img.Img +import net.imglib2.img.array.ArrayImgs +import net.imglib2.realtransform.AffineTransform3D +import net.imglib2.type.numeric.ARGBType +import net.imglib2.type.numeric.NumericType +import net.imglib2.type.numeric.RealType +import net.imglib2.type.numeric.integer.UnsignedByteType +import net.imglib2.view.Views +import org.joml.Quaternionf +import org.joml.Vector3f +import org.joml.Vector4f +import org.scijava.Context +import org.scijava.`object`.ObjectService +import org.scijava.display.Display +import org.scijava.event.EventHandler +import org.scijava.event.EventService +import org.scijava.io.IOService +import org.scijava.log.LogLevel +import org.scijava.log.LogService +import org.scijava.menu.MenuService +import org.scijava.plugin.Parameter +import org.scijava.service.SciJavaService +import org.scijava.thread.ThreadService +import org.scijava.util.ColorRGB +import org.scijava.util.Colors +import org.scijava.util.VersionUtils +import sc.iview.commands.demo.animation.ParticleDemo +import sc.iview.event.NodeActivatedEvent +import sc.iview.event.NodeAddedEvent +import sc.iview.event.NodeChangedEvent +import sc.iview.event.NodeRemovedEvent +import sc.iview.process.MeshConverter +import sc.iview.ui.CustomPropertyUI +import sc.iview.ui.MainWindow +import sc.iview.ui.SwingMainWindow +import sc.iview.ui.TaskManager +import tpietzsch.example2.VolumeViewerOptions +import java.awt.event.WindowListener +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.FloatBuffer +import java.nio.file.Path +import java.util.* +import java.util.concurrent.Future +import java.util.function.Consumer +import java.util.function.Function +import java.util.function.Predicate +import java.util.stream.Collectors +import kotlin.collections.ArrayList +import kotlin.collections.HashMap +import kotlin.math.cos +import kotlin.math.sin +import kotlin.system.measureTimeMillis + +/** + * Main SciView class. + * + * @author Kyle Harrington + */ +// we suppress unused warnings here because @Parameter-annotated fields +// get updated automatically by SciJava. +class SciView : SceneryBase, CalibratedRealInterval { + val sceneryPanel = arrayOf(null) + + /* + * Return the default floor object + *//* + * Set the default floor object + */ + /** + * The floor that orients the user in the scene + */ + var floor: Node? = null + protected var vrActive = false + + /** + * The primary camera/observer in the scene + */ + var camera: Camera? = null + set(value) { + field = value + setActiveObserver(field) + } + + lateinit var controls: Controls + val targetArcball: AnimatedCenteringBeforeArcBallControl + get() = controls.targetArcball + + val currentScene: Scene + get() = scene + + /** + * Geometry/Image information of scene + */ + private lateinit var axes: Array + + @Parameter + private lateinit var log: LogService + + @Parameter + private lateinit var menus: MenuService + + @Parameter + private lateinit var io: IOService + + @Parameter + private lateinit var eventService: EventService + + @Parameter + private lateinit var lutService: LUTService + + @Parameter + private lateinit var threadService: ThreadService + + @Parameter + private lateinit var objectService: ObjectService + + @Parameter + private lateinit var unitService: UnitService + + private lateinit var imageToVolumeMap: HashMap + + /** + * Queue keeps track of the currently running animations + */ + private var animations: Queue>? = null + + /** + * Animation pause tracking + */ + private var animating = false + + /** + * This tracks the actively selected Node in the scene + */ + var activeNode: Node? = null + private set + + /* + * Return the SciJava Display that contains SciView + *//* + * Set the SciJava Display + */ var display: Display<*>? = null + + /** + * Return the current SceneryJPanel. This is necessary for custom context menus + * @return panel the current SceneryJPanel + */ + var lights: ArrayList? = null + private set + private val notAbstractNode: Predicate = Predicate { node: Node -> !(node is Camera || node is Light || node === floor) } + var isClosed = false + internal set + + private val notAbstractBranchingFunction = Function { node: Node -> node.children.stream().filter(notAbstractNode).collect(Collectors.toList()) } + + val taskManager = TaskManager() + + // If true, then when a new node is added to the scene, the camera will refocus on this node by default + var centerOnNewNodes = false + + // If true, then when a new node is added the thread will block until the node is added to the scene. This is required for + // centerOnNewNodes + var blockOnNewNodes = false + private var headlight: PointLight? = null + + lateinit var mainWindow: MainWindow + + constructor(context: Context) : super("SciView", 1280, 720, false, context) { + context.inject(this) + } + + constructor(applicationName: String?, windowWidth: Int, windowHeight: Int) : super(applicationName!!, windowWidth, windowHeight, false) + + fun publicGetInputHandler(): InputHandler { + return inputHandler!! + } + + /** + * Toggle video recording with scenery's video recording mechanism + * Note: this video recording may skip frames because it is asynchronous + */ + fun toggleRecordVideo() { + if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie() else (renderer as VulkanRenderer).recordMovie() + } + + /** + * Toggle video recording with scenery's video recording mechanism + * Note: this video recording may skip frames because it is asynchronous + * + * @param filename destination for saving video + * @param overwrite should the file be replaced, otherwise a unique incrementing counter will be appended + */ + fun toggleRecordVideo(filename: String?, overwrite: Boolean) { + if (renderer is OpenGLRenderer) (renderer as OpenGLRenderer).recordMovie(filename!!, overwrite) else (renderer as VulkanRenderer).recordMovie(filename!!, overwrite) + } + + /** + * See [Controls.stashControls]. + */ + fun stashControls() { + controls.stashControls() + } + + /** + * See [Controls.restoreControls] and [Controls.stashControls]. + */ + fun restoreControls() { + controls.restoreControls() + } + + internal fun setRenderer(newRenderer: Renderer) { + renderer = newRenderer + } + + + /** + * Reset the scene to initial conditions + */ + fun reset() { + // Initialize the 3D axes + axes = arrayOf( + DefaultLinearAxis(DefaultAxisType("X", true), "um", 1.0), + DefaultLinearAxis(DefaultAxisType("Y", true), "um", 1.0), + DefaultLinearAxis(DefaultAxisType("Z", true), "um", 1.0) + ) + + // Remove everything except camera + val toRemove = getSceneNodes { n: Node? -> n !is Camera } + for (n in toRemove) { + deleteNode(n, false) + } + + imageToVolumeMap = HashMap() + + // Setup camera + if (camera == null) { + camera = DetachedHeadCamera() + (camera as DetachedHeadCamera).position = Vector3f(0.0f, 1.65f, 0.0f) + scene.addChild(camera as DetachedHeadCamera) + } + camera!!.spatial().position = Vector3f(0.0f, 1.65f, 5.0f) + camera!!.perspectiveCamera(50.0f, windowWidth, windowHeight, 0.1f, 1000.0f) + + // Setup lights + val tetrahedron = arrayOfNulls(4) + tetrahedron[0] = Vector3f(1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[1] = Vector3f(-1.0f, 0f, -1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[2] = Vector3f(0.0f, 1.0f, 1.0f / Math.sqrt(2.0).toFloat()) + tetrahedron[3] = Vector3f(0.0f, -1.0f, 1.0f / Math.sqrt(2.0).toFloat()) + lights = ArrayList() + for (i in 0..3) { // TODO allow # initial lights to be customizable? + val light = PointLight(150.0f) + light.spatial().position = tetrahedron[i]!!.mul(25.0f) + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + light.intensity = 1.0f + lights!!.add(light) + //camera.addChild( light ); + scene.addChild(light) + } + + // Make a headlight for the camera + headlight = PointLight(150.0f) + headlight!!.spatial().position = Vector3f(0f, 0f, -1f).mul(25.0f) + headlight!!.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + headlight!!.intensity = 0.5f + headlight!!.name = "headlight" + val lightSphere = Icosphere(1.0f, 2) + headlight!!.addChild(lightSphere) + lightSphere.material().diffuse = headlight!!.emissionColor + lightSphere.material().specular = headlight!!.emissionColor + lightSphere.material().ambient = headlight!!.emissionColor + lightSphere.material().wireframe = true + lightSphere.visible = false + //lights.add( light ); + camera!!.nearPlaneDistance = 0.01f + camera!!.farPlaneDistance = 1000.0f + camera!!.addChild(headlight!!) + floor = InfinitePlane() //new Box( new Vector3f( 500f, 0.2f, 500f ) ); + (floor as InfinitePlane).type = InfinitePlane.Type.Grid + (floor as Node).name = "Floor" + scene.addChild(floor as Node) + } + + /** + * Initialization of SWING and scenery. Also triggers an initial population of lights/camera in the scene + */ + override fun init() { + val logLevel = System.getProperty("scenery.LogLevel", "info") + log.level = LogLevel.value(logLevel) + LogbackUtils.setLogLevel(null, logLevel) + System.getProperties().stringPropertyNames().forEach(Consumer { name: String -> + if (name.startsWith("scenery.LogLevel")) { + LogbackUtils.setLogLevel("", System.getProperty(name, "info")) + } + }) + + // determine imagej-launcher version and to disable Vulkan if XInitThreads() fix + // is not deployed + try { + val launcherClass = Class.forName("net.imagej.launcher.ClassLauncher") + var versionString = VersionUtils.getVersion(launcherClass) + if (versionString != null && getPlatform() == ExtractsNatives.Platform.LINUX) { + versionString = versionString.substring(0, 5) + val launcherVersion = Version(versionString) + val nonWorkingVersion = Version("4.0.5") + if (launcherVersion.compareTo(nonWorkingVersion) <= 0 + && !java.lang.Boolean.parseBoolean(System.getProperty("sciview.DisableLauncherVersionCheck", "false"))) { + logger.info("imagej-launcher version smaller or equal to non-working version ($versionString vs. 4.0.5), disabling Vulkan as rendering backend. Disable check by setting 'scenery.DisableLauncherVersionCheck' system property to 'true'.") + System.setProperty("scenery.Renderer", "OpenGLRenderer") + } else { + logger.info("imagej-launcher version bigger that non-working version ($versionString vs. 4.0.5), all good.") + } + } + } catch (cnfe: ClassNotFoundException) { + // Didn't find the launcher, so we're probably good. + logger.info("imagej-launcher not found, not touching renderer preferences.") + } + + animations = LinkedList() + mainWindow = SwingMainWindow(this) + controls = Controls(this) + + imageToVolumeMap = HashMap() + } + + fun toggleSidebar(): Boolean { + return mainWindow.toggleSidebar() + + } + + private fun initializeInterpreter() { + mainWindow.initializeInterpreter() + } + + /* + * Completely close the SciView window + cleanup + */ + fun closeWindow() { + mainWindow.close() + dispose() + } + + /* + * Return true if the scene has been initialized + */ + val isInitialized: Boolean + get() = sceneInitialized() + + /** + * Place the scene into the center of camera view, and zoom in/out such + * that the whole scene is in the view (everything would be visible if it + * would not be potentially occluded). + */ + fun fitCameraToScene() { + centerOnNode(scene) + //TODO: smooth zoom in/out VLADO vlado Vlado + } + + /** + * Place the scene into the center of camera view. + */ + fun centerOnScene() { + centerOnNode(scene) + } + /* + * Get the InputHandler that is managing mouse, input, VR controls, etc. + */ + val sceneryInputHandler: InputHandler + get() = inputHandler!! + + /* + * Return a bounding box around a subgraph of the scenegraph + */ + fun getSubgraphBoundingBox(n: Node): OrientedBoundingBox? { + val predicate = Function> { node: Node -> node.children } + return getSubgraphBoundingBox(n, predicate) + } + + /* + * Return a bounding box around a subgraph of the scenegraph + */ + fun getSubgraphBoundingBox(n: Node, branchFunction: Function>): OrientedBoundingBox? { + if (n.boundingBox == null && n.children.size != 0) { + return n.getMaximumBoundingBox().asWorld() + } + val branches = branchFunction.apply(n) + if (branches.isEmpty()) { + return if (n.boundingBox == null) null else n.boundingBox!!.asWorld() + } + var bb = n.getMaximumBoundingBox() + for (c in branches) { + val cBB = getSubgraphBoundingBox(c, branchFunction) + if (cBB != null) bb = bb.expand(bb, cBB) + } + return bb + } + + /** + * Place the active node into the center of camera view. + */ + fun centerOnActiveNode() { + if (activeNode == null) return + centerOnNode(activeNode) + } + + /** + * Place the specified node into the center of camera view. + */ + fun centerOnNode(currentNode: Node?) { + if (currentNode == null) { + log.info("Cannot center on null node.") + return + } + + //center the on the same spot as ArcBall does + centerOnPosition(currentNode.getMaximumBoundingBox().getBoundingSphere().origin) + } + + /** + * Center the camera on the specified Node + */ + fun centerOnPosition(currentPos: Vector3f?) { + controls.centerOnPosition(currentPos) + } + + /** + * Activate the node, and center the view on it. + * @param n + * @return the currently active node + */ + fun setActiveCenteredNode(n: Node?): Node? { + //activate... + val ret = setActiveNode(n) + //...and center it + ret?.let { centerOnNode(it) } + return ret + } + + //a couple of shortcut methods to readout controls params + fun getFPSSpeedSlow(): Float { + return controls.getFPSSpeedSlow() + } + + fun getFPSSpeedFast(): Float { + return controls.getFPSSpeedFast() + } + + fun getFPSSpeedVeryFast(): Float { + return controls.getFPSSpeedVeryFast() + } + + fun getMouseSpeed(): Float { + return controls.getMouseSpeed() + } + + fun getMouseScrollSpeed(): Float { + return controls.getMouseScrollSpeed() + } + + //a couple of setters with scene sensible boundary checks + fun setFPSSpeedSlow(slowSpeed: Float) { + controls.setFPSSpeedSlow(slowSpeed) + } + + fun setFPSSpeedFast(fastSpeed: Float) { + controls.setFPSSpeedFast(fastSpeed) + } + + fun setFPSSpeedVeryFast(veryFastSpeed: Float) { + controls.setFPSSpeedVeryFast(veryFastSpeed) + } + + fun setFPSSpeed(newBaseSpeed: Float) { + controls.setFPSSpeed(newBaseSpeed) + } + + fun setMouseSpeed(newSpeed: Float) { + controls.setMouseSpeed(newSpeed) + } + + fun setMouseScrollSpeed(newSpeed: Float) { + controls.setMouseScrollSpeed(newSpeed) + } + + fun setObjectSelectionMode() { + controls.setObjectSelectionMode() + } + + /* + * Set the action used during object selection + */ + fun setObjectSelectionMode(selectAction: Function3?) { + controls.setObjectSelectionMode(selectAction) + } + + fun showContextNodeChooser(x: Int, y: Int) { + mainWindow.showContextNodeChooser(x,y) + } + + /* + * Initial configuration of the scenery InputHandler + * This is automatically called and should not be used directly + */ + override fun inputSetup() { + log.info("Running InputSetup") + controls.inputSetup() + } + + /** + * Add a box at the specified position with specified size, color, and normals on the inside/outside + * @param position position to put the box + * @param size size of the box + * @param color color of the box + * @param inside are normals inside the box? + * @return the Node corresponding to the box + */ + @JvmOverloads + fun addBox(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), size: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, + inside: Boolean = false, block: Box.() -> Unit = {}): Box { + val box = Box(size, inside) + box.spatial().position = position + box.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + return addNode(box, block = block) + } + + /** + * Add a unit sphere at a given [position] with given [radius] and [color]. + * @return the Node corresponding to the sphere + */ + @JvmOverloads + fun addSphere(position: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), radius: Float = 1f, color: ColorRGB = DEFAULT_COLOR, block: Sphere.() -> Unit = {}): Sphere { + val sphere = Sphere(radius, 20) + sphere.spatial().position = position + sphere.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + + return addNode(sphere, block = block) + } + + /** + * Add a Cylinder at the given position with radius, height, and number of faces/segments + * @param position position of the cylinder + * @param radius radius of the cylinder + * @param height height of the cylinder + * @param num_segments number of segments to represent the cylinder + * @return the Node corresponding to the cylinder + */ + fun addCylinder(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cylinder.() -> Unit = {}): Cylinder { + val cyl = Cylinder(radius, height, num_segments) + cyl.spatial().position = position + return addNode(cyl, block = block) + } + + /** + * Add a Cone at the given position with radius, height, and number of faces/segments + * @param position position to put the cone + * @param radius radius of the cone + * @param height height of the cone + * @param num_segments number of segments used to represent cone + * @return the Node corresponding to the cone + */ + fun addCone(position: Vector3f, radius: Float, height: Float, num_segments: Int, block: Cone.() -> Unit = {}): Cone { + val cone = Cone(radius, height, num_segments, Vector3f(0.0f, 0.0f, 1.0f)) + cone.spatial().position = position + return addNode(cone, block = block) + } + + /** + * Add a line from start to stop with the given color + * @param start start position of line + * @param stop stop position of line + * @param color color of line + * @return the Node corresponding to the line + */ + @JvmOverloads + fun addLine(start: Vector3f = Vector3f(0.0f, 0.0f, 0.0f), stop: Vector3f = Vector3f(1.0f, 1.0f, 1.0f), color: ColorRGB = DEFAULT_COLOR, block: Line.() -> Unit = {}): Line { + return addLine(arrayOf(start, stop), color, 0.1, block) + } + + /** + * Add a multi-segment line that goes through the supplied points with a single color and edge width + * @param points points along line including first and terminal points + * @param color color of line + * @param edgeWidth width of line segments + * @return the Node corresponding to the line + */ + @JvmOverloads + fun addLine(points: Array, color: ColorRGB, edgeWidth: Double, block: Line.() -> Unit = {}): Line { + val line = Line(points.size) + for (pt in points) { + line.addPoint(pt) + } + line.edgeWidth = edgeWidth.toFloat() + line.material { + ambient = Vector3f(1.0f, 1.0f, 1.0f) + diffuse = Utils.convertToVector3f(color) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + line.spatial().position = points[0] + return addNode(line, block = block) + } + + /** + * Add a PointLight source at the origin + * @return a Node corresponding to the PointLight + */ + @JvmOverloads + fun addPointLight(block: PointLight.() -> Unit = {}): PointLight { + val light = PointLight(5.0f) + light.material { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Vector3f(0.0f, 1.0f, 0.0f) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + light.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + lights!!.add(light) + return addNode(light, block = block) + } + + /** + * Position all lights that were initialized by default around the scene in a circle at Y=0 + */ + fun surroundLighting() { + val bb = getSubgraphBoundingBox(scene, notAbstractBranchingFunction) + val (c, r) = bb!!.getBoundingSphere() + // Choose a good y-position, then place lights around the cross-section through this plane + val y = 0f + for (k in lights!!.indices) { + val light = lights!![k] + val x = (c.x() + r * cos(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() + val z = (c.y() + r * sin(if (k == 0) 0.0 else Math.PI * 2 * (k.toFloat() / lights!!.size.toFloat()))).toFloat() + light.lightRadius = 2 * r + light.spatial().position = Vector3f(x, y, z) + } + } + + @Throws(IOException::class) + fun openDirTiff(source: Path, onlyFirst: Int? = null) + { + val v = Volume.fromPath(source, hub, onlyFirst) + v.name = "volume" + v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = true + + v.spatial().wantsComposeModel = true + v.spatial().updateWorld(true) +// System.out.println("v.model: " + v.model) + addChild(v) +// System.out.println("v.getDimensions: "+ v.getDimensions()) +// +// System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) +// System.out.println("v.world.matrix: " + v.spatial().world) + } + + data class PointInTrack( + val t: Int, + val loc: Vector3f, + val cellId: Long, + val parentId: Long, + val nodeScore: Float, + val edgeScore: Float + ) + + data class Track( + val track: List, + val trackId: Int + ) + + @Throws(IOException::class) + fun openTrackFile(file: File) + { + val lines = file.readLines() + var track = ArrayList() + val tracks = ArrayList() + val separator = "," + + var lastTrackId = -1 + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val t = tokens[0].toInt() + val z = tokens[1].toFloat() -2000f + val y = tokens[2].toFloat() -800f + val x = tokens[3].toFloat() -1300f + val cellId = tokens[4].toLong() + val parentId = tokens[5].toLong() + val trackId = tokens[6].toInt() + val nodeScore = tokens[7].toFloat() + val edgeScore = tokens[8].toFloat()/45.0f + + val currentPointInTrack = PointInTrack( + t, + Vector3f(x,y,z), + cellId, + parentId, + nodeScore, + edgeScore + ) + if(lastTrackId != trackId) + { + lastTrackId = trackId + val sortedTrack = track.sortedBy { it.t } + tracks.add(Track(sortedTrack, trackId)) + + track.clear() + } + track.add(currentPointInTrack) + } + val timeCost = measureTimeMillis { + addTracks(tracks) + } + println("time: $timeCost") + } + + fun addTracks(tracks: ArrayList) + { + val rng = Random(17) + for(track in tracks) + { + if(track.trackId > 10) + { + continue + } + System.out.println("add track: "+ track.trackId.toString() ) + val master = Cylinder(0.1f, 1.0f, 10) +// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Vector3f(0.05f, 0f, 0f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + mInstanced.name = "TrackID-${track.trackId}" + mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } + addNode(mInstanced) + + var cnt = 0 + val a = rng.nextFloat() + val b = rng.nextFloat() + track.track.windowed(2,1).forEach { pair -> + cnt = cnt + 1 + val element = mInstanced.addInstance() + element.name ="EdgeID-$cnt" + element.instancedProperties["Color"] = {Vector4f( a,b,pair[0].edgeScore, 1.0f)} + element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) + //mInstanced.instances.add(element) + + } + } + + } + + + /** + * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud + * @param source string of a data source + * @throws IOException + */ + @Suppress("UNCHECKED_CAST") + @Throws(IOException::class) + fun open(source: String) { + if (source.endsWith(".xml", ignoreCase = true)) { + addNode(fromXML(source, hub, VolumeViewerOptions())) + return + } else if (source.takeLast(4).equals(".pdb", true)) { + val protein = Protein.fromFile(source) + val ribbon = RibbonDiagram(protein) + ribbon.spatial().position = Vector3f(0f, 0f, 0f) + addNode(ribbon) + return + } else if (source.endsWith(".stl", ignoreCase = true)) { + val stlReader = STLMeshIO() + addMesh(stlReader.open(source)) + return + } else if (source.endsWith(".ply", ignoreCase = true)) { + val plyReader = PLYMeshIO() + addMesh(plyReader.open(source)) + return + } + val data = io.open(source) + if (data is Mesh) + addMesh(data) + else if (data is graphics.scenery.Mesh) + addMesh(data) + else if (data is PointCloud) + addPointCloud(data) + else if (data is Dataset) + addVolume(data) + else if (data is RandomAccessibleInterval<*>) + addVolume(data as RandomAccessibleInterval>, source) + else if (data is List<*>) { + val list = data + require(!list.isEmpty()) { "Data source '$source' appears empty." } + val element = list[0] + if (element is RealLocalizable) { + // NB: For now, we assume all elements will be RealLocalizable. + // Highly likely to be the case, barring antagonistic importers. + val points = list as List + addPointCloud(points, source) + } else { + val type = if (element == null) "" else element.javaClass.name + throw IllegalArgumentException("Data source '" + source + // + "' contains elements of unknown type '" + type + "'") + } + } else { + val type = if (data == null) "" else data.javaClass.name + throw IllegalArgumentException("Data source '" + source + // + "' contains data of unknown type '" + type + "'") + } + } + + /** + * Add the given points to the scene as a PointCloud with a given name + * @param points points to use in a PointCloud + * @param name name of the PointCloud + * @return + */ + @JvmOverloads + fun addPointCloud(points: Collection, + name: String? = "PointCloud", + pointSize : Float = 1.0f, + block: PointCloud.() -> Unit = {}): PointCloud { + val flatVerts = FloatArray(points.size * 3) + var k = 0 + for (point in points) { + flatVerts[k * 3] = point.getFloatPosition(0) + flatVerts[k * 3 + 1] = point.getFloatPosition(1) + flatVerts[k * 3 + 2] = point.getFloatPosition(2) + k++ + } + val pointCloud = PointCloud(pointSize, name!!) + val vBuffer: FloatBuffer = BufferUtils.allocateFloat(flatVerts.size * 4) + val nBuffer: FloatBuffer = BufferUtils.allocateFloat(0) + vBuffer.put(flatVerts) + vBuffer.flip() + pointCloud.geometry().vertices = vBuffer + pointCloud.geometry().normals = nBuffer + pointCloud.geometry().indices = BufferUtils.allocateInt(0) + pointCloud.spatial().position = Vector3f(0f, 0f, 0f) + + pointCloud.setupPointCloud() + return addNode(pointCloud, block = block) + } + + /** + * Add a PointCloud to the scene + * @param pointCloud existing PointCloud to add to scene + * @return a Node corresponding to the PointCloud + */ + @JvmOverloads + fun addPointCloud(pointCloud: PointCloud, block: PointCloud.() -> Unit = {}): PointCloud { + pointCloud.setupPointCloud() + pointCloud.spatial().position = Vector3f(0f, 0f, 0f) + return addNode(pointCloud, block = block) + } + + /** + * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true + * @param n node to add to scene + * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services + * @return a Node corresponding to the Node + */ + @JvmOverloads + fun addNode(n: N, activePublish: Boolean = true, block: N.() -> Unit = {}): N { + n?.let { + it.block() + scene.addChild(it) + objectService.addObject(n) + if (blockOnNewNodes) { + Utils.blockWhile({ this.find(n.name) == null }, 20) + //System.out.println("find(name) " + find(n.getName()) ); + } + // Set new node as active and centered? + setActiveNode(n) + if (centerOnNewNodes) { + centerOnNode(n) + } + if (activePublish) { + eventService.publish(NodeAddedEvent(n)) + } + } + return n + } + + /** + * Add a scenery Mesh to the scene + * @param scMesh scenery mesh to add to scene + * @return a Node corresponding to the mesh + */ + fun addMesh(scMesh: graphics.scenery.Mesh): graphics.scenery.Mesh { + scMesh.ifMaterial { + ambient = Vector3f(1.0f, 0.0f, 0.0f) + diffuse = Vector3f(0.0f, 1.0f, 0.0f) + specular = Vector3f(1.0f, 1.0f, 1.0f) + } + scMesh.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + objectService.addObject(scMesh) + return addNode(scMesh) + } + + /** + * Add an ImageJ mesh to the scene + * @param mesh net.imagej.mesh to add to scene + * @return a Node corresponding to the mesh + */ + fun addMesh(mesh: Mesh): graphics.scenery.Mesh { + val scMesh = MeshConverter.toScenery(mesh) + return addMesh(scMesh) + } + + /** + * [Deprecated: use deleteNode] + * Remove a Mesh from the scene + * @param scMesh mesh to remove from scene + */ + fun removeMesh(scMesh: graphics.scenery.Mesh?) { + scene.removeChild(scMesh!!) + } + + /** + * Set the currently active node + * @param n existing node that should become active focus of this SciView + * @return the currently active node + */ + fun setActiveNode(n: Node?): Node? { + if (activeNode === n) return activeNode + activeNode = n + targetArcball.target = { n?.getMaximumBoundingBox()?.getBoundingSphere()?.origin ?: Vector3f(0.0f, 0.0f, 0.0f) } + mainWindow.selectNode(activeNode) + eventService.publish(NodeActivatedEvent(activeNode)) + return activeNode + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeAdded(event: NodeAddedEvent?) { + mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeRemoved(event: NodeRemovedEvent?) { + mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeChanged(event: NodeChangedEvent?) { + // TODO: Check if rebuilding the tree is necessary here, otherwise this costs a lot of performance + //mainWindow.rebuildSceneTree() + } + + @Suppress("UNUSED_PARAMETER") + @EventHandler + protected fun onNodeActivated(event: NodeActivatedEvent?) { + // TODO: add listener code for node activation, if necessary + // NOTE: do not update property window here, this will lead to a loop. + } + + fun toggleInspectorWindow() { + toggleSidebar() + } + + @Suppress("UNUSED_PARAMETER") + fun setInspectorWindowVisibility(visible: Boolean) { +// inspector.setVisible(visible); +// if( visible ) +// mainSplitPane.setDividerLocation(getWindowWidth()/4 * 3); +// else +// mainSplitPane.setDividerLocation(getWindowWidth()); + } + + @Suppress("UNUSED_PARAMETER") + fun setInterpreterWindowVisibility(visible: Boolean) { +// interpreterPane.getComponent().setVisible(visible); +// if( visible ) +// interpreterSplitPane.setDividerLocation(getWindowHeight()/10 * 6); +// else +// interpreterSplitPane.setDividerLocation(getWindowHeight()); + } + + /** + * Create an animation thread with the given fps speed and the specified action + * @param fps frames per second at which this action should be run + * @param action Runnable that contains code to run fps times per second + * @return a Future corresponding to the thread + */ + @Synchronized + fun animate(fps: Int, action: Runnable): Future<*> { + // TODO: Make animation speed less laggy and more accurate. + val delay = 1000 / fps + val thread = threadService.run { + while (animating) { + action.run() + try { + Thread.sleep(delay.toLong()) + } catch (e: InterruptedException) { + break + } + } + } + animations!!.add(thread) + animating = true + return thread + } + + /** + * Stop all animations + */ + @Synchronized + fun stopAnimation() { + animating = false + while (!animations!!.isEmpty()) { + animations!!.peek().cancel(true) + animations!!.remove() + } + } + + /** + * Take a screenshot and save it to the default scenery location + */ + fun takeScreenshot() { + renderer!!.screenshot() + } + + /** + * Take a screenshot and save it to the specified path + * @param path path for saving the screenshot + */ + fun takeScreenshot(path: String?, overwrite: Boolean = false) { + renderer!!.screenshot(path!!, overwrite = overwrite) + } + + /** + * Take a screenshot and return it as an Img + * @return an Img of type UnsignedByteType + */ + val screenshot: Img + get() { + val screenshot = getSceneryRenderer()?.requestScreenshot() ?: throw IllegalStateException("No renderer present, cannot create screenshot") + return ArrayImgs.unsignedBytes(screenshot.data!!, screenshot.width.toLong(), screenshot.height.toLong(), 4L) + } + + /** + * Take a screenshot and return it as an Img + * @return an Img of type UnsignedByteType + */ + val aRGBScreenshot: Img + get() { + return Utils.convertToARGB(screenshot) + } + + /** + * @param name The name of the node to find. + * @return the node object or null, if the node has not been found. + */ + fun find(name: String): Node? { + val n = scene.find(name) + if (n == null) { + logger.warn("Node with name $name not found.") + } + return n + } + + /** + * @return an array of all nodes in the scene except Cameras and PointLights + */ + val sceneNodes: Array + get() = getSceneNodes { n: Node? -> n !is Camera && n !is PointLight } + + /** + * Get a list of nodes filtered by filter predicate + * @param filter, a predicate that filters the candidate nodes + * @return all nodes that match the predicate + */ + fun getSceneNodes(filter: Predicate): Array { + return scene.children.filter{ filter.test(it) }.toTypedArray() + } + + /** + * @return an array of all Node's in the scene + */ + val allSceneNodes: Array + get() = getSceneNodes { _: Node? -> true } + + /** + * Delete the current active node + */ + fun deleteActiveNode() { + deleteNode(activeNode) + } + + /** + * Delete a specified node and control whether the event is published + * @param node node to delete from scene + * @param activePublish whether the deletion should be published + */ + @JvmOverloads + fun deleteNode(node: Node?, activePublish: Boolean = true) { + if(node is Volume) { + node.volumeManager.remove(node) + val toRemove = ArrayList() + for( entry in imageToVolumeMap.entries ) { + if( entry.value == node ) { + toRemove.add(entry.key) + } + } + for(entry in toRemove) { + imageToVolumeMap.remove(entry) + } + } + + for (child in node!!.children) { + deleteNode(child, activePublish) + } + objectService.removeObject(node) + node.parent?.removeChild(node) + if (activeNode == node) { + setActiveNode(null) + } + //maintain consistency + if( activePublish ) { + eventService.publish(NodeRemovedEvent(node)) + } + } + + /** + * Dispose the current scenery renderer, hub, and other scenery things + */ + fun dispose() { + val objs: List = objectService.getObjects(Node::class.java) + for (obj in objs) { + objectService.removeObject(obj) + } + scijavaContext!!.service(SciViewService::class.java).close(this) + close() + // if scijavaContext was not created by ImageJ, then system exit + if( objectService.getObjects(Utils.SciviewStandalone::class.java).size > 0 ) { + log.info("Was running as sciview standalone, shutting down JVM") + System.exit(0) + } + } + + override fun close() { + super.close() + } + + /** + * Move the current active camera to the specified position + * @param position position to move the camera to + */ + fun moveCamera(position: FloatArray) { + camera?.spatial()?.position = Vector3f(position[0], position[1], position[2]) + } + + /** + * Move the current active camera to the specified position + * @param position position to move the camera to + */ + fun moveCamera(position: DoubleArray) { + camera?.spatial()?.position = Vector3f(position[0].toFloat(), position[1].toFloat(), position[2].toFloat()) + } + + /** + * Get the current application name + * @return a String of the application name + */ + fun getName(): String { + return applicationName + } + + /** + * Add a child to the scene. you probably want addNode + * @param node node to add as a child to the scene + */ + fun addChild(node: Node) { + scene.addChild(node) + } + + /** + * Add a Dataset to the scene as a volume. Voxel resolution and name are extracted from the Dataset itself + * @param image image to add as a volume + * @return a Node corresponding to the Volume + */ + @JvmOverloads + fun addVolume(image: Dataset, block: Volume.() -> Unit = {}): Volume { + val voxelDims = FloatArray(image.numDimensions()) + + for (d in voxelDims.indices) { + val inValue = image.axis(d).averageScale(0.0, 1.0) + if (image.axis(d).unit() == null) { + voxelDims[d] = inValue.toFloat() + } else { + val imageAxisUnit = image.axis(d).unit().replace("µ", "u") + val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") + + voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() + } + } + + logger.info("Adding with ${voxelDims.joinToString(",")}") + val v = addVolume(image, voxelDims, block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a Dataset as a Volume with the specified voxel dimensions + * @param image image to add as a volume + * @param voxelDimensions dimensions of voxels in volume + * @return a Node corresponding to the Volume + */ + @JvmOverloads + @Suppress("UNCHECKED_CAST") + fun addVolume(image: Dataset, voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { + val v = addVolume(image.imgPlus as RandomAccessibleInterval>, image.name ?: "Volume", + *voxelDimensions, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a RandomAccessibleInterval to the image + * @param image image to add as a volume + * @param pixel type of image + * @return a Node corresponding to the volume + */ + @JvmOverloads + fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", block: Volume.() -> Unit = {}): Volume { + val v = addVolume(image, name, 1f, 1f, 1f, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add a RandomAccessibleInterval to the image + * @param image image to add as a volume + * @param pixel type of image + * @return a Node corresponding to the volume + */ + fun > addVolume(image: RandomAccessibleInterval, voxelDimensions: FloatArray, block: Volume.() -> Unit): Volume { + val v = addVolume(image, "volume", *voxelDimensions, block = block) + imageToVolumeMap[image] = v + return v + } + + /** + * Add an IterableInterval as a Volume + * @param image + * @param + * @return a Node corresponding to the Volume + */ + @Suppress("UNCHECKED_CAST") + @Throws(Exception::class) + fun > addVolume(image: IterableInterval): Volume { + return if (image is RandomAccessibleInterval<*>) { + val v = addVolume(image as RandomAccessibleInterval>, "Volume") + imageToVolumeMap[image] = v + v + } else { + throw Exception("Unsupported Volume type:$image") + } + } + + /** + * Add an IterableInterval as a Volume + * @param image image to add as a volume + * @param name name of image + * @param pixel type of image + * @return a Node corresponding to the Volume + */ + @Suppress("UNCHECKED_CAST") + @Throws(Exception::class) + fun > addVolume(image: IterableInterval, name: String = "Volume"): Volume { + return if (image is RandomAccessibleInterval<*>) { + val v = addVolume(image as RandomAccessibleInterval>, name, 1f, 1f, 1f) + imageToVolumeMap[image] = v + v + } else { + throw Exception("Unsupported Volume type:$image") + } + } + + /** + * Set the colormap using an ImageJ LUT name + * @param n node to apply colormap to + * @param lutName name of LUT according to imagej LUTService + */ + fun setColormap(n: Node, lutName: String) { + try { + setColormap(n, lutService.loadLUT(lutService.findLUTs()[lutName])) + } catch (e: IOException) { + e.printStackTrace() + } + } + + /** + * Set the ColorMap of node n to the supplied colorTable + * @param n node to apply colortable to + * @param colorTable ColorTable to use + */ + fun setColormap(n: Node, colorTable: ColorTable) { + val copies = 16 + val byteBuffer = ByteBuffer.allocateDirect( + 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture + val tmp = ByteArray(4 * colorTable.length) + for (k in 0 until colorTable.length) { + for (c in 0 until colorTable.componentCount) { + // TODO this assumes numBits is 8, could be 16 + tmp[4 * k + c] = colorTable[c, k].toByte() + } + if (colorTable.componentCount == 3) { + tmp[4 * k + 3] = 255.toByte() + } + } + for (i in 0 until copies) { + byteBuffer.put(tmp) + } + byteBuffer.flip() + n.metadata["sciviewColormap"] = colorTable + if (n is Volume) { + n.colormap = Colormap.fromColorTable(colorTable) + n.geometryOrNull()?.dirty = true + n.spatial().needsUpdate = true + } + } + + /** + * Adss a SourceAndConverter to the scene. + * + * @param sac The SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + fun > addVolume(sac: SourceAndConverter, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + val sources: MutableList> = ArrayList() + sources.add(sac) + + val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) + imageToVolumeMap[sources] = v + imageToVolumeMap[sac] = v + return v + } + + /** + * Add an IterableInterval to the image with the specified voxelDimensions and name + * This version of addVolume does most of the work + * @param image image to add as a volume + * @param name name of image + * @param voxelDimensions dimensions of voxel in volume + * @param pixel type of image + * @return a Node corresponding to the Volume + */ + @JvmOverloads + fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", + vararg voxelDimensions: Float, block: Volume.() -> Unit = {}): Volume { + //log.debug( "Add Volume " + name + " image: " + image ); + val dimensions = LongArray(image.numDimensions()) + image.dimensions(dimensions) + val minPt = LongArray(image.numDimensions()) + + // Get type at min point + val imageRA = image.randomAccess() + image.min(minPt) + imageRA.setPosition(minPt) + val voxelType = imageRA.get()!!.createVariable() + val converterSetups: ArrayList = ArrayList() + val stacks = AxisOrder.splitInputStackIntoSourceStacks(image, AxisOrder.getAxisOrder(AxisOrder.DEFAULT, image, false)) + val sourceTransform = AffineTransform3D() + val sources: ArrayList> = ArrayList>() + var numTimepoints = 1 + for (stack in stacks) { + var s: Source + if (stack.numDimensions() > 3) { + numTimepoints = (stack.max(3) + 1).toInt() + s = RandomAccessibleIntervalSource4D(stack, voxelType, sourceTransform, name) + } else { + s = RandomAccessibleIntervalSource(stack, voxelType, sourceTransform, name) + } + val source = BigDataViewer.wrapWithTransformedSource( + SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) + converterSetups.add(BigDataViewer.createConverterSetup(source, setupId.getAndIncrement())) + sources.add(source) + } + val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) + v.metadata.set("RandomAccessibleInterval", image) + imageToVolumeMap[image] = v + return v + } + + /** + * Adds a SourceAndConverter to the scene. + * + * This method actually instantiates the volume. + * + * @param sources The list of SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + @Suppress("UNCHECKED_CAST") + fun > addVolume(sources: List>, + converterSetups: ArrayList, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + var timepoints = numTimepoints + var cacheControl: CacheControl? = null + +// RandomAccessibleInterval image = +// ((RandomAccessibleIntervalSource4D) sources.get(0).getSpimSource()). +// .getSource(0, 0); + val image = sources[0].spimSource.getSource(0, 0) + if (image is VolatileView<*, *>) { + val viewData = (image as VolatileView>).volatileViewData + cacheControl = viewData.cacheControl + } + val dimensions = LongArray(image.numDimensions()) + image.dimensions(dimensions) + val minPt = LongArray(image.numDimensions()) + + // Get type at min point + val imageRA = image.randomAccess() + image.min(minPt) + imageRA.setPosition(minPt) + val voxelType = imageRA.get()!!.createVariable() as T + println("addVolume " + image.numDimensions() + " interval " + image as Interval) + + //int numTimepoints = 1; + if (image.numDimensions() > 3) { + timepoints = image.dimension(3).toInt() + } + val ds = RAISource(voxelType, sources, converterSetups, timepoints, cacheControl) + val options = VolumeViewerOptions() + val v: Volume = RAIVolume(ds, options, hub) + v.name = name + v.metadata["sources"] = sources + v.metadata["VoxelDimensions"] = voxelDimensions + v.spatial().scale = Vector3f(1.0f, voxelDimensions[1]/voxelDimensions[0], voxelDimensions[2]/voxelDimensions[0]) * v.pixelToWorldRatio * 10.0f + val tf = v.transferFunction + val rampMin = 0f + val rampMax = 0.1f + tf.clear() + tf.addControlPoint(0.0f, 0.0f) + tf.addControlPoint(rampMin, 0.0f) + tf.addControlPoint(1.0f, rampMax) + val bg = BoundingGrid() + bg.node = v + + imageToVolumeMap[image] = v + return addNode(v, block = block) + } + + /** + * Adds a SourceAndConverter to the scene. + * + * @param sources The list of SourceAndConverter to add + * @param name Name of the dataset + * @param voxelDimensions Array with voxel dimensions. + * @param Type of the dataset. + * @return THe node corresponding to the volume just added. + */ + @JvmOverloads + fun > addVolume(sources: List>, + numTimepoints: Int, + name: String = "Volume", + vararg voxelDimensions: Float, + block: Volume.() -> Unit = {}): Volume { + var setupId = 0 + val converterSetups = ArrayList() + for (source in sources) { + converterSetups.add(BigDataViewer.createConverterSetup(source, setupId++)) + } + val v = addVolume(sources, converterSetups, numTimepoints, name, *voxelDimensions, block = block) + imageToVolumeMap[sources] = v + return v + } + + /** + * Get the Volume that corresponds to an image if one exists + * @param image an image of any type (e.g. IterableInterval, RAI, SourceAndConverter) + * @return a Volume corresponding to the input image + */ + fun getVolumeFromImage(image: Any): Volume? { + if( image in imageToVolumeMap ) + return imageToVolumeMap[image] + return null + } + + /** + * Update a volume with the given IterableInterval. + * This method actually populates the volume + * @param image image to update into volume + * @param name name of image + * @param voxelDimensions dimensions of voxel in volume + * @param v existing volume to update + * @param pixel type of image + * @return a Node corresponding to the input volume + */ + @Suppress("UNCHECKED_CAST") + fun > updateVolume(image: IterableInterval, name: String, + voxelDimensions: FloatArray, v: Volume): Volume { + val sacs = v.metadata["sources"] as List>? + val source = sacs!![0].spimSource.getSource(0, 0) // hard coded to timepoint and mipmap 0 + val sCur = Views.iterable(source).cursor() + val iCur = image.cursor() + while (sCur.hasNext()) { + sCur.fwd() + iCur.fwd() + sCur.get()!!.set(iCur.get()) + } + v.name = name + v.metadata["VoxelDimensions"] = voxelDimensions + v.volumeManager.notifyUpdate(v) + v.volumeManager.requestRepaint() + //v.getCacheControls().clear(); + //v.setDirty( true ); + v.spatial().needsUpdate = true + //v.setNeedsUpdateWorld( true ); + return v + } + + /** + * + * @return whether PushMode is currently active + */ + fun getPushMode(): Boolean { + return renderer!!.pushMode + } + + /** + * Set the status of PushMode, which only updates the render panel when there is a change in the scene + * @param push true if push mode should be used + * @return current PushMode status + */ + fun setPushMode(push: Boolean): Boolean { + renderer!!.pushMode = push + return renderer!!.pushMode + } + + protected fun finalize() { + stopAnimation() + } + + fun getScenerySettings(): Settings { + return settings + } + + fun getSceneryStatistics(): Statistics { + return stats + } + + fun getSceneryRenderer(): Renderer? { + return renderer + } + + /** + * Enable VR rendering + */ + fun toggleVRRendering() { + vrActive = !vrActive + val cam = scene.activeObserver as? DetachedHeadCamera ?: return + var ti: TrackerInput? = null + var hmdAdded = false + if (!hub.has(SceneryElement.HMDInput)) { + try { + val hmd = OpenVRHMD(false, true) + if (hmd.initializedAndWorking()) { + hub.add(SceneryElement.HMDInput, hmd) + ti = hmd + } else { + logger.warn("Could not initialise VR headset, just activating stereo rendering.") + } + hmdAdded = true + } catch (e: Exception) { + logger.error("Could not add OpenVRHMD: $e") + } + } else { + ti = hub.getWorkingHMD() + } + if (vrActive && ti != null) { + cam.tracker = ti + } else { + cam.tracker = null + } + renderer!!.pushMode = false + // we need to force reloading the renderer as the HMD might require device or instance extensions + if (renderer is VulkanRenderer && hmdAdded) { + replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) + (renderer as VulkanRenderer).toggleVR() + while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { + logger.debug("Waiting for renderer reinitialisation") + try { + Thread.sleep(200) + } catch (e: InterruptedException) { + e.printStackTrace() + } + } + } else { + renderer!!.toggleVR() + } +// renderer!!.toggleVR() + } + + /** + * Set the rotation of Node N by generating a quaternion from the supplied arguments + * @param n node to set rotation for + * @param x x coord of rotation quat + * @param y y coord of rotation quat + * @param z z coord of rotation quat + * @param w w coord of rotation quat + */ + fun setRotation(n: Node, x: Float, y: Float, z: Float, w: Float) { + n.spatialOrNull()?.rotation = Quaternionf(x, y, z, w) + } + + fun setScale(n: Node, x: Float, y: Float, z: Float) { + n.spatialOrNull()?.scale = Vector3f(x, y, z) + } + + @Suppress("UNUSED_PARAMETER") + fun setColor(n: Node, x: Float, y: Float, z: Float, w: Float) { + val col = Vector3f(x, y, z) + n.ifMaterial { + ambient = col + diffuse = col + specular = col + } + } + + fun setPosition(n: Node, x: Float, y: Float, z: Float) { + n.spatialOrNull()?.position = Vector3f(x, y, z) + } + + fun addWindowListener(wl: WindowListener?) { + (mainWindow as? SwingMainWindow)?.addWindowListener(wl) + } + + override fun axis(i: Int): CalibratedAxis? { + return axes[i] + } + + override fun axes(calibratedAxes: Array) { + axes = calibratedAxes + } + + override fun setAxis(calibratedAxis: CalibratedAxis, i: Int) { + axes[i] = calibratedAxis + } + + override fun realMin(i: Int): Double { + return Double.NEGATIVE_INFINITY + } + + override fun realMin(doubles: DoubleArray) { + for (i in doubles.indices) { + doubles[i] = Double.NEGATIVE_INFINITY + } + } + + override fun realMin(realPositionable: RealPositionable) { + for (i in 0 until realPositionable.numDimensions()) { + realPositionable.move(Double.NEGATIVE_INFINITY, i) + } + } + + override fun realMax(i: Int): Double { + return Double.POSITIVE_INFINITY + } + + override fun realMax(doubles: DoubleArray) { + for (i in doubles.indices) { + doubles[i] = Double.POSITIVE_INFINITY + } + } + + override fun realMax(realPositionable: RealPositionable) { + for (i in 0 until realPositionable.numDimensions()) { + realPositionable.move(Double.POSITIVE_INFINITY, i) + } + } + + override fun numDimensions(): Int { + return axes.size + } + + fun setActiveObserver(screenshotCam: Camera?) { + scene.activeObserver = screenshotCam + } + + fun getActiveObserver(): Camera? { + return scene.activeObserver + } + + /** + * Return a list of all nodes that match a given predicate function + * @param nodeMatchPredicate, returns true if a node is a match + * @return list of nodes that match the predicate + */ + fun findNodes(nodeMatchPredicate: Function1): List { + return scene.discover(scene, nodeMatchPredicate, false) + } + + /* + * Convenience function for getting a string of info about a Node + */ + fun nodeInfoString(n: Node): String { + return "Node name: " + n.name + " Node type: " + n.nodeType + " To String: " + n + } + + /** + * Triggers the inspector tree to be completely rebuilt/refreshed. + */ + fun requestPropEditorRefresh() { + eventService.publish(NodeChangedEvent(scene)) + } + + /** + * Triggers the inspector to rebuild/refresh the given node. + * @param n Root of the subtree to get rebuilt/refreshed. + */ + fun requestPropEditorRefresh(n: Node?) { + eventService.publish(NodeChangedEvent(n)) + } + + fun attachCustomPropertyUIToNode(node: Node, ui: CustomPropertyUI) { + node.metadata["sciview-inspector-${ui.module.info.name}"] = ui + } + + fun getAvailableServices() { + println(scijavaContext!!.serviceIndex) + } + + companion object { + //bounds for the controls + const val FPSSPEED_MINBOUND_SLOW = 0.01f + const val FPSSPEED_MAXBOUND_SLOW = 30.0f + const val FPSSPEED_MINBOUND_FAST = 0.2f + const val FPSSPEED_MAXBOUND_FAST = 600f + const val FPSSPEED_MINBOUND_VERYFAST = 10f + const val FPSSPEED_MAXBOUND_VERYFAST = 2000f + + const val MOUSESPEED_MINBOUND = 0.1f + const val MOUSESPEED_MAXBOUND = 3.0f + const val MOUSESCROLL_MINBOUND = 0.3f + const val MOUSESCROLL_MAXBOUND = 10.0f + + @JvmField + val DEFAULT_COLOR: ColorRGB = Colors.LIGHTGRAY + + /** + * Static launching method + * + * @return a newly created SciView + */ + @JvmStatic + @Throws(Exception::class) + fun create(): SciView { + xinitThreads() + val context = Context(ImageJService::class.java, SciJavaService::class.java, SCIFIOService::class.java, + ThreadService::class.java, ObjectService::class.java, LogService::class.java, MenuService::class.java, + IOService::class.java, EventService::class.java, LUTService::class.java, UnitService::class.java, + DatasetIOService::class.java) + val objectService = context.getService(ObjectService::class.java) + objectService.addObject(Utils.SciviewStandalone()) + val sciViewService = context.service(SciViewService::class.java) + return sciViewService.orCreateActiveSciView + } + + /** + * Static launching method + * DEPRECATED use SciView.create() instead + * + * @return a newly created SciView + */ + @Deprecated("Please use SciView.create() instead.", replaceWith = ReplaceWith("SciView.create()")) + @Throws(Exception::class) + fun createSciView(): SciView { + return create() + } + } +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index f632343c..76cbfefd 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -1,674 +1,674 @@ -package sc.iview.commands.demo.advanced - -import bdv.util.BdvFunctions -import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew -import graphics.scenery.numerics.OpenSimplexNoise -import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.* -import org.scijava.Context -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class EyeTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) - val calibrationTarget = Icosphere(0.02f, 2) - val laser = Cylinder(0.005f, 0.2f, 10) - - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - val confidenceThreshold = 0.60f - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - calibrationTarget.visible = false - calibrationTarget.material { - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.camera!!.addChild(calibrationTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addChild(hedgehogs) - - val eyeFrames = Mesh("eyeFrames") - val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) - val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) - left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) - left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) - right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) - eyeFrames.addChild(left) - eyeFrames.addChild(right) - - sciview.addChild(eyeFrames) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - pupilTracker.subscribeFrames { eye, texture -> - if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { - return@subscribeFrames - } - - val node = if(eye == 1) { - left - } else { - right - } - - val stream = ByteArrayInputStream(texture) - val image = ImageIO.read(stream) - val data = (image.raster.dataBuffer as DataBufferByte).data - - node.ifMaterial {textures["diffuse"] = Texture( - Vector3i(image.width, image.height, 1), - 3, - UnsignedByteType(), - BufferUtils.allocateByteAndPut(data) - ) } - - lastFrame = System.nanoTime() - } - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogs.visible = false - hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogs.children.removeAt(hedgehogs.children.size-1) - volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> - volume.removeChild(lastTrack) - } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}") - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", duration = 1000) - dumpHedgehog() - addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupCalibration() - - } - - - private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { - val startCalibration = ClickBehaviour { _, _ -> - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - pupilTracker.gazeConfidenceThreshold = confidenceThreshold - if (!pupilTracker.isCalibrated) { - pupilTracker.onCalibrationInProgress = { - cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) - } - - pupilTracker.onCalibrationFailed = { - cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) - } - - pupilTracker.onCalibrationSuccess = { - cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) -// cam.children.find { it.name == "debugBoard" }?.visible = true - - for (i in 0 until 20) { - referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} - Thread.sleep(100) - referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} - Thread.sleep(30) - } - - hmd.removeBehaviour("start_calibration") - hmd.removeKeyBinding("start_calibration") - - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } else { - addHedgehog() - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - } - tracking = !tracking - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } - playing = true - } - - pupilTracker.unsubscribeFrames() - sciview.deleteNode(sciview.find("eyeFrames")) - - log.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) - pupilTracker.calibrate(cam, hmd, - generateReferenceData = true, - calibrationTarget = calibrationTarget) - - pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { - //NEW - PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> - if (gaze.confidence > confidenceThreshold) { - val p = gaze.gazePoint() - referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.position = p - (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" - - val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() - - if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) - } - } - } - -// else -> {gaze-> } - } - - log.info("Calibration routine done.") - } - - // bind calibration start to menu key on controller - - } - } - hmd.addBehaviour("start_calibration", startCalibration) - hmd.addKeyBinding("start_calibration", keybindingCalibration) - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - val spine = (hedgehogs.children.last() as InstancedNode).addInstance() - spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = true - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) -// System.out.println(intersection); - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.2f } - - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - var lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = if(lastHedgehog == null) { - val m = Cylinder(3f, 1.0f, 10) - m.ifMaterial { - ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - roughness = 1.0f - metallic = 0.0f - cullingMode = Material.CullingMode.None - } - m.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(m) - mInstanced - } else { - null - } - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - if(master != null) { - val element = master.addInstance() - element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) - element.parent = volume - master.instances.add(element) - } - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - master?.let { volume.addChild(it) } - - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.numerics.OpenSimplexNoise +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class EyeTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.camera!!.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addChild(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.ifMaterial {textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.allocateByteAndPut(data) + ) } + + lastFrame = System.nanoTime() + } + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogs.children.removeAt(hedgehogs.children.size-1) + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}") + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", duration = 1000) + dumpHedgehog() + addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupCalibration() + + } + + + private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { + val startCalibration = ClickBehaviour { _, _ -> + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated) { + pupilTracker.onCalibrationInProgress = { + cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) +// cam.children.find { it.name == "debugBoard" }?.visible = true + + for (i in 0 until 20) { + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} + Thread.sleep(30) + } + + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } else { + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + log.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + //NEW + PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) + } + } + } + +// else -> {gaze-> } + } + + log.info("Calibration routine done.") + } + + // bind calibration start to menu key on controller + + } + } + hmd.addBehaviour("start_calibration", startCalibration) + hmd.addKeyBinding("start_calibration", keybindingCalibration) + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = true + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) +// System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + var lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = if(lastHedgehog == null) { + val m = Cylinder(3f, 1.0f, 10) + m.ifMaterial { + ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + roughness = 1.0f + metallic = 0.0f + cullingMode = Material.CullingMode.None + } + m.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(m) + mInstanced + } else { + null + } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + if(master != null) { + val element = master.addInstance() + element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + element.parent = volume + master.instances.add(element) + } + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + master?.let { volume.addChild(it) } + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index ee3db7dc..cf06a144 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -1,435 +1,435 @@ -package graphics.scenery.bionictracking - -import graphics.scenery.Icosphere -import graphics.scenery.Scene -import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f -import org.joml.Vector3f -import org.joml.Vector4f -import org.joml.Matrix4f -import org.joml.Quaternionf -import graphics.scenery.utils.LazyLogger -import graphics.scenery.utils.extensions.* -import org.scijava.log.LogService -import org.scijava.plugin.Parameter -import org.slf4j.LoggerFactory -import java.io.File -import kotlin.math.abs -import kotlin.math.pow -import kotlin.math.sqrt - -/** - * - * - * @author Ulrik Günther - */ -class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { - - private val logger by LazyLogger() - - val timepoints = LinkedHashMap>() - - var avgConfidence = 0.0f - private set - var totalSampleCount = 0 - private set - - data class Track( - val points: List>, - val confidence: Float - ) - - init { - logger.info("Starting analysis with ${spines.size} spines") - - spines.forEach { spine -> - val timepoint = spine.timepoint - val current = timepoints[timepoint] - - if(current == null) { - timepoints[timepoint] = arrayListOf(spine) - } else { - current.add(spine) - } - - avgConfidence += spine.confidence - totalSampleCount++ - } - - avgConfidence /= totalSampleCount - } - - private fun localMaxima(list: List): List> = - list.windowed(3, 1).mapIndexed { index, l -> - val left = l[0] - val center = l[1] - val right = l[2] - - // we have a match at center - if(left - center < 0 && center - right > 0) { - index + 1 to center - } else { - null - } - }.filterNotNull() - - data class SpineGraphVertex(val timepoint: Int, - val position: Vector3f, - val worldPosition: Vector3f, - val index: Int, - val value: Float, - val metadata : SpineMetadata, - var previous: SpineGraphVertex? = null, - var next: SpineGraphVertex? = null) { - - fun distance(): Float { - val n = next - return if(n != null) { - val t = (n.worldPosition - this.worldPosition) - sqrt(t.x*t.x + t.y*t.y + t.z*t.z) - } else { - 0.0f - } - } - - fun drop() { - previous?.next = next - next?.previous = previous - } - - override fun toString() : String { - return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" - } - } - - fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) -// fun Iterable.avg() = (this.map { it}.sum() / this.count()) - - fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { - val cross = forward.cross(this) - val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) - - val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) - - return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) - } - - fun run(): Track? { - logger.info("run track analysis") - val startingThreshold = 0.002f - val localMaxThreshold = 0.001f - val zscoreThreshold = 2.0f - val removeTooFarThreshold = 5.0f - - if(timepoints.isEmpty()) { - logger.info("timepoints is empty") - return null - } - - - //step1: find the startingPoint by using startingthreshold - val startingPoint = timepoints.entries.firstOrNull { entry -> - entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } - } ?: return null - - logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") - -// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) - - timepoints.filter { it.key > startingPoint.key } - .forEach { timepoints.remove(it.key) } - - logger.info("${timepoints.size} timepoints left") - - - //step2: find the maxIndices along the spine - val candidates = timepoints.map { tp -> - val vs = tp.value.mapIndexedNotNull { i, spine -> - val maxIndices = localMaxima(spine.samples.filterNotNull()) -// logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") - - if(maxIndices.isNotEmpty()) { -// filter the maxIndices which are too far away - maxIndices.filter { it.first <1200}. - map { index -> -// logger.info(index.toString()) - val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) -// logger.info("i: " + i) -// logger.info("position: " + position) -// logger.info("dimension: "+ dimension) -// logger.info("localToWorld: "+ localToWorld) - val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() -// logger.info("world position: "+ worldPosition) - SpineGraphVertex(tp.key, - position, - worldPosition, - index.first, - index.second, - spine) - - } - } else { - null - } - } - vs - }.flatten() - - - //step3: connect localMaximal points between 2 candidate spines according to the shortest path principle - // get the initial vertex, this one is assumed to always be in front, and have a local max - val initial = candidates.first().filter{it.value>startingThreshold}.first() - logger.info("initial:"+initial) - logger.info("candidates number: "+ candidates.size) - var current = initial - var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> -// System.out.logger.info("time: ${time}") -// logger.info("vs: ${vs}") - val distances = vs - .filter { it.value > localMaxThreshold } - .map { vertex -> - val t = current.worldPosition - vertex.worldPosition - val distance = t.length() -// logger.info("current worldposition:"+ current.worldPosition) -// logger.info("vertex.worldposition"+vertex.worldPosition) - vertex to distance - } - .sortedBy { it.second } - //logger.info("distances.size: "+distances.size) - //logger.info("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) -// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) -// { -// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") -// } -// - val closest = distances.firstOrNull()?.first - if(closest != null && distances.firstOrNull()?.second!! >0) { - current.next = closest - closest.previous = current - current = closest - current - } else { - null - } - }.toMutableList() - - - val beforeCount = shortestPath.size -// System.out.logger.info("before short path:"+ shortestPath.size) - - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - - fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) - - //step4: if some path is longer than multiple average length, it should be removed - while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { - shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - -// logger.info("check which one is removed") -// shortestPath.forEach { -// if(it.distance() >= removeTooFarThreshold * avgPathLength) -// { -// logger.info("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) -// } -// } - } -// - avgPathLength = shortestPath.map { it.distance() }.average().toFloat() - stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - - //step5: remove some edges according to zscoreThreshold - var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - while(remaining > 0) { - val outliers = shortestPath - .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - .map { - val idx = shortestPath.indexOf(it) - listOf(idx-1,idx,idx+1) - }.flatten() - - shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - - //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - - remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } - - shortestPath.windowed(3, 1, partialWindows = true).forEach { - it.getOrNull(0)?.next = it.getOrNull(1) - it.getOrNull(1)?.previous = it.getOrNull(0) - it.getOrNull(1)?.next = it.getOrNull(2) - it.getOrNull(2)?.previous = it.getOrNull(1) - } - logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") - } - - val afterCount = shortestPath.size - logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") -// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") -// logger.info(shortestPath.toString()) - val singlePoints = shortestPath - .groupBy { it.timepoint } - .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } - .filter { - it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f - } - - - logger.info("Returning ${singlePoints.size} points") - - - return Track(singlePoints.map { it.position to it}, avgConfidence) - } - - companion object { - private val logger by LazyLogger(System.getProperty("scenery.LogLevel", "info")) - - fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { - logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val confidence = tokens[1].toFloat() - val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - Vector3f(0.0f), - Vector3f(0.0f), - 0.0f, - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Vector3f(0.0f), - Quaternionf(), - Vector3f(0.0f), - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f()) - } - - private fun String.toVector3f(): Vector3f { -// System.out.logger.info(this) - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - - if (array[0] == "+Inf" || array[0] == "-Inf") - return Vector3f(0.0f,0.0f,0.0f) - - return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) - } - - private fun String.toQuaternionf(): Quaternionf { -// System.out.logger.info(this) - val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} - return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) - } - fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - logger.info("lines number: " + lines.size) - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, matrix4f) - } - - fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { - logger.info("Loading spines from complete CSV at ${csv.absolutePath}") - - val lines = csv.readLines() - val spines = ArrayList(lines.size) - - lines.drop(1).forEach { line -> - val tokens = line.split(separator) - val timepoint = tokens[0].toInt() - val origin = tokens[1].toVector3f() - val direction = tokens[2].toVector3f() - val localEntry = tokens[3].toVector3f() - val localExit = tokens[4].toVector3f() - val localDirection = tokens[5].toVector3f() - val headPosition = tokens[6].toVector3f() - val headOrientation = tokens[7].toQuaternionf() - val position = tokens[8].toVector3f() - val confidence = tokens[9].toFloat() - val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } - - val currentSpine = SpineMetadata( - timepoint, - origin, - direction, - 0.0f, - localEntry, - localExit, - localDirection, - headPosition, - headOrientation, - position, - confidence, - samples) - - spines.add(currentSpine) - } - - return HedgehogAnalysis(spines, Matrix4f()) - } - } -} - -fun main(args: Array) { - val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") -// if(args.isEmpty()) { -// logger.error("Sorry, but a file name is needed.") -// return -// } - - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") -// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) - val analysis = HedgehogAnalysis.fromCSV(file) - val results = analysis.run() - logger.info("Results: \n$results") -} +package graphics.scenery.bionictracking + +import graphics.scenery.Icosphere +import graphics.scenery.Scene +import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f +import org.joml.Vector3f +import org.joml.Vector4f +import org.joml.Matrix4f +import org.joml.Quaternionf +import graphics.scenery.utils.LazyLogger +import graphics.scenery.utils.extensions.* +import org.scijava.log.LogService +import org.scijava.plugin.Parameter +import org.slf4j.LoggerFactory +import java.io.File +import kotlin.math.abs +import kotlin.math.pow +import kotlin.math.sqrt + +/** + * + * + * @author Ulrik Günther + */ +class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { + + private val logger by LazyLogger() + + val timepoints = LinkedHashMap>() + + var avgConfidence = 0.0f + private set + var totalSampleCount = 0 + private set + + data class Track( + val points: List>, + val confidence: Float + ) + + init { + logger.info("Starting analysis with ${spines.size} spines") + + spines.forEach { spine -> + val timepoint = spine.timepoint + val current = timepoints[timepoint] + + if(current == null) { + timepoints[timepoint] = arrayListOf(spine) + } else { + current.add(spine) + } + + avgConfidence += spine.confidence + totalSampleCount++ + } + + avgConfidence /= totalSampleCount + } + + private fun localMaxima(list: List): List> = + list.windowed(3, 1).mapIndexed { index, l -> + val left = l[0] + val center = l[1] + val right = l[2] + + // we have a match at center + if(left - center < 0 && center - right > 0) { + index + 1 to center + } else { + null + } + }.filterNotNull() + + data class SpineGraphVertex(val timepoint: Int, + val position: Vector3f, + val worldPosition: Vector3f, + val index: Int, + val value: Float, + val metadata : SpineMetadata, + var previous: SpineGraphVertex? = null, + var next: SpineGraphVertex? = null) { + + fun distance(): Float { + val n = next + return if(n != null) { + val t = (n.worldPosition - this.worldPosition) + sqrt(t.x*t.x + t.y*t.y + t.z*t.z) + } else { + 0.0f + } + } + + fun drop() { + previous?.next = next + next?.previous = previous + } + + override fun toString() : String { + return "SpineGraphVertex for t=$timepoint, pos=$position,index=$index, worldPos=$worldPosition, value=$value" + } + } + + fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) +// fun Iterable.avg() = (this.map { it}.sum() / this.count()) + + fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { + val cross = forward.cross(this) + val q = Quaternionf(cross.x(), cross.y(), cross.z(), this.dot(forward)) + + val x = sqrt((q.w + sqrt(q.x*q.x + q.y*q.y + q.z*q.z + q.w*q.w)) / 2.0f) + + return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) + } + + fun run(): Track? { + logger.info("run track analysis") + val startingThreshold = 0.002f + val localMaxThreshold = 0.001f + val zscoreThreshold = 2.0f + val removeTooFarThreshold = 5.0f + + if(timepoints.isEmpty()) { + logger.info("timepoints is empty") + return null + } + + + //step1: find the startingPoint by using startingthreshold + val startingPoint = timepoints.entries.firstOrNull { entry -> + entry.value.any { metadata -> metadata.samples.filterNotNull().any { it > startingThreshold } } + } ?: return null + + logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + +// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) + + timepoints.filter { it.key > startingPoint.key } + .forEach { timepoints.remove(it.key) } + + logger.info("${timepoints.size} timepoints left") + + + //step2: find the maxIndices along the spine + val candidates = timepoints.map { tp -> + val vs = tp.value.mapIndexedNotNull { i, spine -> + val maxIndices = localMaxima(spine.samples.filterNotNull()) +// logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + + if(maxIndices.isNotEmpty()) { +// filter the maxIndices which are too far away + maxIndices.filter { it.first <1200}. + map { index -> +// logger.info(index.toString()) + val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) +// logger.info("i: " + i) +// logger.info("position: " + position) +// logger.info("dimension: "+ dimension) +// logger.info("localToWorld: "+ localToWorld) + val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() +// logger.info("world position: "+ worldPosition) + SpineGraphVertex(tp.key, + position, + worldPosition, + index.first, + index.second, + spine) + + } + } else { + null + } + } + vs + }.flatten() + + + //step3: connect localMaximal points between 2 candidate spines according to the shortest path principle + // get the initial vertex, this one is assumed to always be in front, and have a local max + val initial = candidates.first().filter{it.value>startingThreshold}.first() + logger.info("initial:"+initial) + logger.info("candidates number: "+ candidates.size) + var current = initial + var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> +// System.out.logger.info("time: ${time}") +// logger.info("vs: ${vs}") + val distances = vs + .filter { it.value > localMaxThreshold } + .map { vertex -> + val t = current.worldPosition - vertex.worldPosition + val distance = t.length() +// logger.info("current worldposition:"+ current.worldPosition) +// logger.info("vertex.worldposition"+vertex.worldPosition) + vertex to distance + } + .sortedBy { it.second } + //logger.info("distances.size: "+distances.size) + //logger.info("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) +// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) +// { +// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") +// } +// + val closest = distances.firstOrNull()?.first + if(closest != null && distances.firstOrNull()?.second!! >0) { + current.next = closest + closest.previous = current + current = closest + current + } else { + null + } + }.toMutableList() + + + val beforeCount = shortestPath.size +// System.out.logger.info("before short path:"+ shortestPath.size) + + var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + fun zScore(value: Float, m: Float, sd: Float) = ((value - m)/sd) + + //step4: if some path is longer than multiple average length, it should be removed + while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { + shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + +// logger.info("check which one is removed") +// shortestPath.forEach { +// if(it.distance() >= removeTooFarThreshold * avgPathLength) +// { +// logger.info("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) +// } +// } + } +// + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() + stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() + + //step5: remove some edges according to zscoreThreshold + var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + while(remaining > 0) { + val outliers = shortestPath + .filter { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + .map { + val idx = shortestPath.indexOf(it) + listOf(idx-1,idx,idx+1) + }.flatten() + + shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() + + //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") + + remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } + + shortestPath.windowed(3, 1, partialWindows = true).forEach { + it.getOrNull(0)?.next = it.getOrNull(1) + it.getOrNull(1)?.previous = it.getOrNull(0) + it.getOrNull(1)?.next = it.getOrNull(2) + it.getOrNull(2)?.previous = it.getOrNull(1) + } + logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") + } + + val afterCount = shortestPath.size + logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") +// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") +// logger.info(shortestPath.toString()) + val singlePoints = shortestPath + .groupBy { it.timepoint } + .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } + .filter { + it.metadata.direction.dot(it.previous!!.metadata.direction) > 0.5f + } + + + logger.info("Returning ${singlePoints.size} points") + + + return Track(singlePoints.map { it.position to it}, avgConfidence) + } + + companion object { + private val logger by LazyLogger(System.getProperty("scenery.LogLevel", "info")) + + fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { + logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val confidence = tokens[1].toFloat() + val samples = tokens.subList(2, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + Vector3f(0.0f), + Vector3f(0.0f), + 0.0f, + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Vector3f(0.0f), + Quaternionf(), + Vector3f(0.0f), + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + + private fun String.toVector3f(): Vector3f { +// System.out.logger.info(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + + if (array[0] == "+Inf" || array[0] == "-Inf") + return Vector3f(0.0f,0.0f,0.0f) + + return Vector3f(array[0].toFloat(),array[1].toFloat(),array[2].toFloat()) + } + + private fun String.toQuaternionf(): Quaternionf { +// System.out.logger.info(this) + val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} + return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) + } + fun fromCSVWithMatrix(csv: File, matrix4f: Matrix4f,separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV with Matrix at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + logger.info("lines number: " + lines.size) + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, matrix4f) + } + + fun fromCSV(csv: File, separator: String = ";"): HedgehogAnalysis { + logger.info("Loading spines from complete CSV at ${csv.absolutePath}") + + val lines = csv.readLines() + val spines = ArrayList(lines.size) + + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val timepoint = tokens[0].toInt() + val origin = tokens[1].toVector3f() + val direction = tokens[2].toVector3f() + val localEntry = tokens[3].toVector3f() + val localExit = tokens[4].toVector3f() + val localDirection = tokens[5].toVector3f() + val headPosition = tokens[6].toVector3f() + val headOrientation = tokens[7].toQuaternionf() + val position = tokens[8].toVector3f() + val confidence = tokens[9].toFloat() + val samples = tokens.subList(10, tokens.size - 1).map { it.toFloat() } + + val currentSpine = SpineMetadata( + timepoint, + origin, + direction, + 0.0f, + localEntry, + localExit, + localDirection, + headPosition, + headOrientation, + position, + confidence, + samples) + + spines.add(currentSpine) + } + + return HedgehogAnalysis(spines, Matrix4f()) + } + } +} + +fun main(args: Array) { + val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") +// if(args.isEmpty()) { +// logger.error("Sorry, but a file name is needed.") +// return +// } + + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") +// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) + val analysis = HedgehogAnalysis.fromCSV(file) + val results = analysis.run() + logger.info("Results: \n$results") +} diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index dbd20213..9080e735 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -1,506 +1,506 @@ -package sc.iview.commands.demo.advanced - -import graphics.scenery.* -import graphics.scenery.attribute.material.Material -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.numerics.Random -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import graphics.scenery.utils.LazyLogger -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Volume -import org.joml.* -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.log.LogService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.text.DecimalFormat -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class Test: Command{ - @Parameter - private lateinit var sciview: SciView - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) - //val calibrationTarget = Icosphere(0.02f, 2) - val TestTarget = Icosphere(0.1f, 2) - - val laser = Cylinder(0.005f, 0.2f, 10) - - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - lateinit var point1:Icosphere - lateinit var point2:Icosphere - - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - val confidenceThreshold = 0.60f - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Backward - @Parameter(label = "Volumes per second") - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.addChild(TestTarget) - TestTarget.visible = false - - -// sciview.toggleVRRendering() -// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - shell.name = "shell" - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - volume.visible = false - - point1 = Icosphere(0.1f, 2) - point1.spatial().position = Vector3f(1.858f,2f,8.432f) - point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - sciview.addChild(point1) - - point2 = Icosphere(0.1f, 2) - point2.spatial().position = Vector3f(1.858f, 2f, -10.39f) - point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} - sciview.addChild(point2) - - - val connector = Cylinder.betweenPoints(point1.position, point2.position) - connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(connector) - - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addChild(hedgehogs) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - //println("timepoint: "+ newTimepoint); - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - //dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - setupControllerforTracking() - - } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - volume.visible = true - volume.runRecursive { it.visible = true } - playing = true - tracking = true - //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position - - if(true) - { -// val p = Vector3f(0f,0f,-1f) -// referenceTarget.position = p -// referenceTarget.visible = true - val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() -// - val direction = (pointWorld - headCenter).normalize() - - if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - //System.out.println("tracking!!!!!!!!!!") -// println("direction:"+ direction.toString()) - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - showTrack() - } - Thread.sleep(200) - } - //referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - - - } // bind calibration start to menu key on controller - - } - - private fun showTrack() - { -// val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") - val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-10-19 13.48.51\\Hedgehog_1_2022-10-19 13.49.41.csv") - - var volumeDimensions = volume.getDimensions() - var selfdefineworlfmatrix = volume.spatial().world - // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) -// selfdefineworlfmatrix = Matrix4f( -// 0.015f, 0f, 0f, 0f, -// 0f, -0.015f, 0f, 0f, -// 0f, 0f, 0.045f, 0f, -// -5f, 8f, -2f, 1f -// ) - val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix) - print("volume.getDimensions(): "+ volume.getDimensions()) - print("volume.spatial().world: "+ volume.spatial().world) - print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) - - val track = analysis.run() - - print("flag1") - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - print("flag2") - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - - val mInstanced = InstancedNode(master) - sciview.addNode(mInstanced) - - - print("flag3") - if(track == null) - { - return - } - print("flag4") - track.points.windowed(2, 1).forEach { pair -> - - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first)//direct product - val p1 = Vector3f(pair[1].first) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - - val tp = pair[0].second.timepoint - val pp = Icosphere(0.1f, 1) - pp.name = "trackpoint_${tp}_${pair[0].first.x}_${pair[0].first.y}_${pair[0].first.z}" -// println("volumeDimensions: " + volumeDimensions) -// println("volume.spatial().world: " + volume.spatial().world) - println("the local position of the point is:" + pair[0].first) - println("the world position of the point is: "+ p0w) - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addNode(pp) - } - } - - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - - - val p2 = Vector3f(center).add(temp) - - -// print("center position: " + p1.toString()) -// print("p2 position" + p2.toString()) - -// TestTarget.visible = true -// TestTarget.ifSpatial { position = p2} - - -// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() -// spine.spatial().orientBetweenPoints(p1, p2, true, true) -// spine.visible = true - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) - System.out.println(intersection); - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val nf = DecimalFormat("0.0000") - println("Ray intersects volume at world=${intersection.entry.toString(nf)}/${intersection.exit.toString(nf)} local=${localEntry.toString(nf)}/${localExit.toString(nf)} ") - -// System.out.println("localEntry:"+ localEntry.toString()) -// System.out.println("localExit:" + localExit.toString()) -// val worldpositionEntry = volume.spatial().world.transform((Vector3f(localEntry)).xyzw()).xyz() -// val worldpositionExit = volume.spatial().world.transform((Vector3f(localExit)).xyzw()).xyz() -// System.out.println("worldEntry:"+ worldpositionEntry.toString()) -// System.out.println("worldExit:" + worldpositionExit.toString()) - - - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.002f } - - println("count of samples: "+ count.toString()) -println(samples) - -// spine.metadata["spine"] = metadata -// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ -// fun dumpHedgehog() { -// var lastHedgehog = hedgehogs.children.last() as InstancedNode -// val hedgehogId = hedgehogIds.incrementAndGet() -// -// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() -// val hedgehogFileWriter = hedgehogFile.bufferedWriter() -// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") -// -// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() -// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) -// if(!trackFile.exists()) { -// trackFile.createNewFile() -// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") -// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") -// } -// -// -// val spines = lastHedgehog.instances.mapNotNull { spine -> -// spine.metadata["spine"] as? SpineMetadata -// } -// -// spines.forEach { metadata -> -// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") -// } -// hedgehogFileWriter.close() -// -// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track -// val track = if(existingAnalysis is HedgehogAnalysis.Track) { -// existingAnalysis -// } else { -// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) -// h.run() -// } -// -// if(track == null) { -//// logger.warn("No track returned") -// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) -// return -// } -// -// lastHedgehog.metadata["HedgehogAnalysis"] = track -// lastHedgehog.metadata["Spines"] = spines -// -//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") -// -// val master = if(lastHedgehog == null) { -// val m = Cylinder(3f, 1.0f, 10) -// m.ifMaterial { -// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") -// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) -// roughness = 1.0f -// metallic = 0.0f -// cullingMode = Material.CullingMode.None -// } -// m.name = "Track-$hedgehogId" -// val mInstanced = InstancedNode(m) -// mInstanced -// } else { -// null -// } -// -// val parentId = 0 -// val volumeDimensions = volume.getDimensions() -// -// trackFileWriter.newLine() -// trackFileWriter.newLine() -// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") -// track.points.windowed(2, 1).forEach { pair -> -// if(master != null) { -// val element = master.addInstance() -// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) -// element.parent = volume -// master.instances.add(element) -// } -// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product -// val tp = pair[0].second.timepoint -// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") -// } -// -// master?.let { volume.addChild(it) } -// -// trackFileWriter.close() -// } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.numerics.Random +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.LazyLogger +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.log.LogService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.text.DecimalFormat +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class Test: Command{ + @Parameter + private lateinit var sciview: SciView + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.004f, 2) + //val calibrationTarget = Icosphere(0.02f, 2) + val TestTarget = Icosphere(0.1f, 2) + + val laser = Cylinder(0.005f, 0.2f, 10) + + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var point1:Icosphere + lateinit var point2:Icosphere + + + val hedgehogs = Mesh() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + val confidenceThreshold = 0.60f + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Backward + @Parameter(label = "Volumes per second") + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.addChild(TestTarget) + TestTarget.visible = false + + +// sciview.toggleVRRendering() +// hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + shell.name = "shell" + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + volume.visible = false + + point1 = Icosphere(0.1f, 2) + point1.spatial().position = Vector3f(1.858f,2f,8.432f) + point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} + sciview.addChild(point1) + + point2 = Icosphere(0.1f, 2) + point2.spatial().position = Vector3f(1.858f, 2f, -10.39f) + point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} + sciview.addChild(point2) + + + val connector = Cylinder.betweenPoints(point1.position, point2.position) + connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + sciview.addChild(connector) + + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addChild(hedgehogs) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + //println("timepoint: "+ newTimepoint); + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hedgehog-> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } else { + hedgehogs.children.forEach { hedgehog -> + val hedgehog = hedgehog as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + //dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false +// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, +// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + var hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + tracking = true + //val p = hmd.getPose(TrackedDeviceType.Controller).firstOrNull { it.name == "Controller-3" }?.position + + if(true) + { +// val p = Vector3f(0f,0f,-1f) +// referenceTarget.position = p +// referenceTarget.visible = true + val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() +// + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + //System.out.println("tracking!!!!!!!!!!") +// println("direction:"+ direction.toString()) + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + showTrack() + } + Thread.sleep(200) + } + //referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + + + } // bind calibration start to menu key on controller + + } + + private fun showTrack() + { +// val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-05-25 16.04.52\\Hedgehog_1_2022-05-25 16.06.03.csv") + val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2022-10-19 13.48.51\\Hedgehog_1_2022-10-19 13.49.41.csv") + + var volumeDimensions = volume.getDimensions() + var selfdefineworlfmatrix = volume.spatial().world + // volumeDimensions = Vector3f(700.0f,660.0f,113.0f) +// selfdefineworlfmatrix = Matrix4f( +// 0.015f, 0f, 0f, 0f, +// 0f, -0.015f, 0f, 0f, +// 0f, 0f, 0.045f, 0f, +// -5f, 8f, -2f, 1f +// ) + val analysis = HedgehogAnalysis.fromCSVWithMatrix(file,selfdefineworlfmatrix) + print("volume.getDimensions(): "+ volume.getDimensions()) + print("volume.spatial().world: "+ volume.spatial().world) + print("selfdefineworlfmatrix: "+ selfdefineworlfmatrix) + + val track = analysis.run() + + print("flag1") + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + print("flag2") + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + sciview.addNode(mInstanced) + + + print("flag3") + if(track == null) + { + return + } + print("flag4") + track.points.windowed(2, 1).forEach { pair -> + + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first)//direct product + val p1 = Vector3f(pair[1].first) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + + val tp = pair[0].second.timepoint + val pp = Icosphere(0.1f, 1) + pp.name = "trackpoint_${tp}_${pair[0].first.x}_${pair[0].first.y}_${pair[0].first.z}" +// println("volumeDimensions: " + volumeDimensions) +// println("volume.spatial().world: " + volume.spatial().world) + println("the local position of the point is:" + pair[0].first) + println("the world position of the point is: "+ p0w) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addNode(pp) + } + } + + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + + + val p2 = Vector3f(center).add(temp) + + +// print("center position: " + p1.toString()) +// print("p2 position" + p2.toString()) + +// TestTarget.visible = true +// TestTarget.ifSpatial { position = p2} + + +// val spine = (hedgehogs.children.last() as InstancedNode).addInstance() +// spine.spatial().orientBetweenPoints(p1, p2, true, true) +// spine.visible = true + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + System.out.println(intersection); + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val nf = DecimalFormat("0.0000") + println("Ray intersects volume at world=${intersection.entry.toString(nf)}/${intersection.exit.toString(nf)} local=${localEntry.toString(nf)}/${localExit.toString(nf)} ") + +// System.out.println("localEntry:"+ localEntry.toString()) +// System.out.println("localExit:" + localExit.toString()) +// val worldpositionEntry = volume.spatial().world.transform((Vector3f(localEntry)).xyzw()).xyz() +// val worldpositionExit = volume.spatial().world.transform((Vector3f(localExit)).xyzw()).xyz() +// System.out.println("worldEntry:"+ worldpositionEntry.toString()) +// System.out.println("worldExit:" + worldpositionExit.toString()) + + + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.002f } + + println("count of samples: "+ count.toString()) +println(samples) + +// spine.metadata["spine"] = metadata +// spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ +// fun dumpHedgehog() { +// var lastHedgehog = hedgehogs.children.last() as InstancedNode +// val hedgehogId = hedgehogIds.incrementAndGet() +// +// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() +// val hedgehogFileWriter = hedgehogFile.bufferedWriter() +// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") +// +// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() +// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) +// if(!trackFile.exists()) { +// trackFile.createNewFile() +// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") +// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") +// } +// +// +// val spines = lastHedgehog.instances.mapNotNull { spine -> +// spine.metadata["spine"] as? SpineMetadata +// } +// +// spines.forEach { metadata -> +// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") +// } +// hedgehogFileWriter.close() +// +// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track +// val track = if(existingAnalysis is HedgehogAnalysis.Track) { +// existingAnalysis +// } else { +// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) +// h.run() +// } +// +// if(track == null) { +//// logger.warn("No track returned") +// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) +// return +// } +// +// lastHedgehog.metadata["HedgehogAnalysis"] = track +// lastHedgehog.metadata["Spines"] = spines +// +//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") +// +// val master = if(lastHedgehog == null) { +// val m = Cylinder(3f, 1.0f, 10) +// m.ifMaterial { +// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") +// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) +// roughness = 1.0f +// metallic = 0.0f +// cullingMode = Material.CullingMode.None +// } +// m.name = "Track-$hedgehogId" +// val mInstanced = InstancedNode(m) +// mInstanced +// } else { +// null +// } +// +// val parentId = 0 +// val volumeDimensions = volume.getDimensions() +// +// trackFileWriter.newLine() +// trackFileWriter.newLine() +// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") +// track.points.windowed(2, 1).forEach { pair -> +// if(master != null) { +// val element = master.addInstance() +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) +// element.parent = volume +// master.instances.add(element) +// } +// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product +// val tp = pair[0].second.timepoint +// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") +// } +// +// master?.let { volume.addChild(it) } +// +// trackFileWriter.close() +// } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 9f71631a..3b077144 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -1,636 +1,636 @@ -package sc.iview.commands.demo.advanced - -import bdv.util.BdvFunctions -import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.xyz -import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction -import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType -import org.joml.* -import org.scijava.Context -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.UIService -import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO -import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.* -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import org.scijava.ui.behaviour.DragBehaviour -import sc.iview.commands.demo.animation.ParticleDemo - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRControllerTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.04f, 2) - val testTarget1 = Icosphere(0.01f, 2) - val testTarget2 = Icosphere(0.04f, 2) - val laser = Cylinder(0.0025f, 1f, 20) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - lateinit var rightController: TrackedDevice - - var hedgehogsList = mutableListOf() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) - laser.material().metallic = 0.0f - laser.material().roughness = 1.0f - laser.visible = false - sciview.addNode(laser) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(referenceTarget) - - testTarget1.visible = false - testTarget1.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(testTarget1) - - - testTarget2.visible = false - testTarget2.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.addNode(testTarget2) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume -// volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") -// if(device.role == TrackerRole.RightHand) { -// rightController = device -// log.info("rightController is found, its location is in ${rightController.position}") -// } -// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) - //dumpHedgehog() - //addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupControllerforTracking() - - } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - println("setupControllerforTracking") - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - tracking = false - dumpHedgehog() - println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) - } else { - addHedgehog() - println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - tracking = true - } - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } - playing = true - - println("test") - - while(true) - { - if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) - { - //println("null") - continue - } - else - { - rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! - - if (rightController.model?.spatialOrNull() == null) { - //println("spatial null") - } - else - { - val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() - val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() - - println(headCenter.toString()) - println(pointWorld.toString()) - testTarget1.visible = true - testTarget1.ifSpatial { position = headCenter} - - testTarget2.visible = true - testTarget2.ifSpatial { position = pointWorld} - - laser.visible = true - laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) - - referenceTarget.visible = true - referenceTarget.ifSpatial { position = pointWorld} - - val direction = (pointWorld - headCenter).normalize() - if (tracking) { - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - } - } - - } - - } - - } - - - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) - - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.02f } - //println("cnt: " + count.toString()) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - master.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(master) - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - sciview.addNode(mInstanced) - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - //mInstanced.instances.add(element) - val pp = Icosphere(0.01f, 1) - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } +package sc.iview.commands.demo.advanced + +import bdv.util.BdvFunctions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.ShaderType +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.numerics.Random +import graphics.scenery.textures.Texture +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.FinalInterval +import net.imglib2.Localizable +import net.imglib2.RandomAccessibleInterval +import net.imglib2.img.array.ArrayImgs +import net.imglib2.position.FunctionRandomAccessible +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.Context +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.UIService +import org.scijava.ui.behaviour.ClickBehaviour +import org.scijava.widget.FileWidget +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.awt.image.DataBufferByte +import java.io.BufferedWriter +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BiConsumer +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI +import net.imglib2.img.Img +import net.imglib2.view.Views +import org.lwjgl.openvr.OpenVR +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import org.scijava.ui.behaviour.DragBehaviour +import sc.iview.commands.demo.animation.ParticleDemo + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRControllerTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.04f, 2) + val testTarget1 = Icosphere(0.01f, 2) + val testTarget2 = Icosphere(0.04f, 2) + val laser = Cylinder(0.0025f, 1f, 20) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + lateinit var rightController: TrackedDevice + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = false + var direction = PlaybackDirection.Forward + var volumesPerSecond = 4 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + laser.material().diffuse = Vector3f(5.0f, 0.0f, 0.02f) + laser.material().metallic = 0.0f + laser.material().roughness = 1.0f + laser.visible = false + sciview.addNode(laser) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(referenceTarget) + + testTarget1.visible = false + testTarget1.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget1) + + + testTarget2.visible = false + testTarget2.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.addNode(testTarget2) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume +// volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") +// if(device.role == TrackerRole.RightHand) { +// rightController = device +// log.info("rightController is found, its location is in ${rightController.position}") +// } +// rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-1")!! + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + + }, + confirmAction = { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) + //dumpHedgehog() + //addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + + setupControllerforTracking() + + } + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + println("setupControllerforTracking") + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + dumpHedgehog() + println("before dumphedgehog: "+ hedgehogsList.last().instances.size.toString()) + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } + playing = true + + println("test") + + while(true) + { + if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) + { + //println("null") + continue + } + else + { + rightController = hmd.getTrackedDevices(TrackedDeviceType.Controller).get("Controller-2")!! + + if (rightController.model?.spatialOrNull() == null) { + //println("spatial null") + } + else + { + val headCenter = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-0.1f).xyzw()).xyz() + val pointWorld = Matrix4f(rightController.model?.spatialOrNull()?.world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + + println(headCenter.toString()) + println(pointWorld.toString()) + testTarget1.visible = true + testTarget1.ifSpatial { position = headCenter} + + testTarget2.visible = true + testTarget2.ifSpatial { position = pointWorld} + + laser.visible = true + laser.spatial().orientBetweenPoints(headCenter, pointWorld,true,true) + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = pointWorld} + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + } + + } + + } + + } + + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) + + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.02f } + //println("cnt: " + count.toString()) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val master = Cylinder(0.1f, 1.0f, 10) + master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + master.name = "Track-$hedgehogId" + val mInstanced = InstancedNode(master) + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + sciview.addNode(mInstanced) + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = mInstanced.addInstance() + val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) + val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() + val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() + element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) + //mInstanced.instances.add(element) + val pp = Icosphere(0.01f, 1) + pp.spatial().position = p0w + pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) + sciview.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + } + } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index 05543587..3d8d6837 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -1,623 +1,623 @@ -package sc.iview.commands.demo.advanced - -import graphics.scenery.* -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.numerics.Random -import graphics.scenery.utils.MaybeIntersects -import graphics.scenery.utils.SystemHelpers -import graphics.scenery.volumes.Volume -import org.joml.* -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin -import org.scijava.ui.behaviour.ClickBehaviour -import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.io.BufferedWriter -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread -import org.scijava.log.LogService -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.behaviours.* -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -import graphics.scenery.utils.extensions.* -import org.scijava.event.EventService -import sc.iview.commands.file.OpenDirofTif -import sc.iview.event.NodeAddedEvent -import sc.iview.event.NodeChangedEvent -import sc.iview.event.NodeRemovedEvent -import sc.iview.event.NodeTaggedEvent -import kotlin.concurrent.fixedRateTimer - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRHeadSetTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView - - @Parameter - private lateinit var log: LogService - - @Parameter - private lateinit var eventService: EventService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.02f, 2) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - var hedgehogsList = mutableListOf() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - private var selectionStorage: Node? = null - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = true - var direction = PlaybackDirection.Backward - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false -// var currentVolume = 0 - - var volumeScaleFactor = 1.0f - - override fun run() { - - sciview.toggleVRRendering() - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - sciview.camera!!.addChild(referenceTarget) - - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) - - volume = sciview.find("volume") as Volume - -// val testtarget = Icosphere(2f, 2) -// volume.addChild(testtarget) -// testtarget.addAttribute(Grabable::class.java,Grabable()) -// testtarget.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = testtarget})) - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } - - thread { - log.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - thread { - dumpHedgehog() - } - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - //LeftController.up.down.left.right - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), - "move_back" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Down), - "move_left" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Left), - "move_right" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Right) - ).forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) - volume.spatial().scale *= Vector3f(volumeScaleFactor) -// println("volumeScaleFactor is " + volumeScaleFactor) -// println("scale is :" + volume.spatial().scale) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) - volume.spatial().scale *= Vector3f(volumeScaleFactor) -// println("volumeScaleFactor is " + volumeScaleFactor) -// println("scale is :" + volume.spatial().scale) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - }, - confirmAction = { - if(hedgehogsList.size != 0) - { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - } - }) - - val playbackDirection = ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - } - - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("playback_direction",playbackDirection) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - - - - hmd.addKeyBinding("skip_to_next", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Right)) // RightController. right - hmd.addKeyBinding("skip_to_prev", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Left)) // RightController. left - hmd.addKeyBinding("faster_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Up)) // RightController. up - hmd.addKeyBinding("slower_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Down)) //RightController. down - hmd.addKeyBinding("play_pause", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Menu)) // LeftController.Menu - hmd.addKeyBinding("playback_direction", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Menu)) //RightController.Menu - hmd.addKeyBinding("delete_hedgehog", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.A)) //RightController.Side - hmd.addKeyBinding("toggle_hedgehog", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.A)) //LeftController.Side - - - - //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) - //left trigger button can validate or delete a track - VRSelect.createAndSet(sciview.currentScene, - hmd, - listOf(OpenVRHMD.OpenVRButton.Trigger), - listOf(TrackerRole.LeftHand), - { n -> - println("the spot ${n.name} is selected") - - //delete the selected node from volume -// volume.runRecursive{it.removeChild(n)} -// eventService.publish(NodeRemovedEvent(n)) - //validate the selected node from volume - eventService.publish(NodeTaggedEvent(n)) - - }, - true) - - - VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) - - VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) - setupControllerforTracking() - } - - - - private fun setupControllerforTracking( keybindingTracking: String = "U") { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - tracking = false - thread { - dumpHedgehog() - println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) - } - } else { - addHedgehog() - println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - tracking = true - } - } - //RightController.trigger - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true - volume.runRecursive { it.visible = true } -// playing = false - - while(true) - { - - val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() - val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() - - referenceTarget.visible = true - referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } - - val direction = (pointWorld - headCenter).normalize() - if (tracking) { - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - } - - Thread.sleep(2) - } - } - - } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) - //println("try to find intersection"); - - if(intersection is MaybeIntersects.Intersection) { - // println("got a intersection") - // get local entry and exit coordinates, and convert to UV coords - val dim = volume.getDimensions() - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - - val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) - val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) - val (samples, localDirection) = volume.sampleRay(entryUV, exitUV) ?: (null to null) - - - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - entryUV, - exitUV, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.002f } -// println("cnt: " + count.toString()) -// println(samples) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - println("do hedgehog Analysis") - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val parent = RichNode() - parent.name = "Track-$hedgehogId" - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = Cylinder(3.0f, 1.0f, 5)//edgeMaster.addInstance() - val p0 = Vector3f(pair[0].first) * Vector3f(volumeDimensions) - val p1 = Vector3f(pair[1].first) * Vector3f(volumeDimensions) - - val tp = pair[0].second.timepoint - - element.spatial().orientBetweenPoints(p0, p1, rescale = true, reposition = true) - element.name = "edge" - element.metadata["Type"] = "edge" - parent.addChild(element) - - val pp = Icosphere(5.0f, 1)//nodeMaster.addInstance() - log.info("Local position: $p0 / $p1") - pp.name = "node-$tp" - pp.metadata["NodeTimepoint"] = tp - pp.metadata["NodePosition"] = p0 - pp.metadata["Type"] = "node" - pp.spatial().position = p0 - - pp.addAttribute(Grabable::class.java, Grabable()) - pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) - pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> - if (device.role == TrackerRole.LeftHand) { - pp.ifSpatial { - position = (device.velocity ?: Vector3f(0.0f)) * 5f + position - eventService.publish(NodeChangedEvent(pp)) - } - } - - })) - parent.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - - volume.addChild(parent) - eventService.publish(NodeAddedEvent(parent)) - - trackFileWriter.close() - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - - command.run(OpenDirofTif::class.java, true, - hashMapOf( - "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), - "onlyFirst" to 10 - )) - .get() - - val argmap = HashMap() - command.run(VRHeadSetTrackingDemo::class.java, true, argmap) - .get() - } - } +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.bionictracking.ConfirmableClickBehaviour +import graphics.scenery.bionictracking.HedgehogAnalysis +import graphics.scenery.bionictracking.SpineMetadata +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.TrackerRole +import graphics.scenery.numerics.Random +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.volumes.Volume +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.BufferedWriter +import java.io.File +import java.io.FileWriter +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.HashMap +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import org.scijava.log.LogService +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.behaviours.* +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.utils.extensions.* +import org.scijava.event.EventService +import sc.iview.commands.file.OpenDirofTif +import sc.iview.event.NodeAddedEvent +import sc.iview.event.NodeChangedEvent +import sc.iview.event.NodeRemovedEvent +import sc.iview.event.NodeTaggedEvent +import kotlin.concurrent.fixedRateTimer + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class VRHeadSetTrackingDemo: Command{ + @Parameter + private lateinit var sciview: SciView + + @Parameter + private lateinit var log: LogService + + @Parameter + private lateinit var eventService: EventService + + lateinit var hmd: OpenVRHMD + val referenceTarget = Icosphere(0.02f, 2) + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + var hedgehogsList = mutableListOf() + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + lateinit var volume: Volume + private var selectionStorage: Node? = null + + enum class PlaybackDirection { + Forward, + Backward + } + + @Volatile var tracking = false + var playing = true + var direction = PlaybackDirection.Backward + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false +// var currentVolume = 0 + + var volumeScaleFactor = 1.0f + + override fun run() { + + sciview.toggleVRRendering() + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + sciview.camera!!.addChild(referenceTarget) + + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + sciview.addChild(shell) + + volume = sciview.find("volume") as Volume + +// val testtarget = Icosphere(2f, 2) +// volume.addChild(testtarget) +// testtarget.addAttribute(Grabable::class.java,Grabable()) +// testtarget.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = testtarget})) + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addChild(it) } + + thread { + log.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + log.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + log.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + inputSetup() + } + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogsList.size>0) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogsList.forEach { hedgehog-> + hedgehog.instances.forEach { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + thread { + dumpHedgehog() + } + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + + } + + fun addHedgehog() { + val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) + hedgehogMaster.visible = false + hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + hedgehogMaster.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) + metallic = 0.01f + roughness = 0.5f + } + var hedgehogInstanced = InstancedNode(hedgehogMaster) + sciview.addNode(hedgehogInstanced) + hedgehogsList.add(hedgehogInstanced) + } + + + fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + //LeftController.up.down.left.right + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), + "move_back" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Down), + "move_left" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Left), + "move_right" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Right) + ).forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = false } + } + cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.PerTimePoint -> { + cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + } + + HedgehogVisibility.Visible -> { + println("the number of hedgehogs: "+ hedgehogsList.size.toString()) + hedgehogsList.forEach { hedgehog -> + println("the number of spines: " + hedgehog.instances.size.toString()) + hedgehog.instances.forEach { it.visible = true } + } + cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) +// println("volumeScaleFactor is " + volumeScaleFactor) +// println("scale is :" + volume.spatial().scale) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) + volume.spatial().scale *= Vector3f(volumeScaleFactor) +// println("volumeScaleFactor is " + volumeScaleFactor) +// println("scale is :" + volume.spatial().scale) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + } else { + cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + } + } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt()) + }, + confirmAction = { + if(hedgehogsList.size != 0) + { + hedgehogsList = hedgehogsList.dropLast(1) as MutableList +// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> +// sciview.removeChild(lastTrack) +// } + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000) + } + }) + + val playbackDirection = ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) + } + + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("playback_direction",playbackDirection) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + + + + hmd.addKeyBinding("skip_to_next", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Right)) // RightController. right + hmd.addKeyBinding("skip_to_prev", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Left)) // RightController. left + hmd.addKeyBinding("faster_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Up)) // RightController. up + hmd.addKeyBinding("slower_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Down)) //RightController. down + hmd.addKeyBinding("play_pause", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Menu)) // LeftController.Menu + hmd.addKeyBinding("playback_direction", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Menu)) //RightController.Menu + hmd.addKeyBinding("delete_hedgehog", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.A)) //RightController.Side + hmd.addKeyBinding("toggle_hedgehog", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.A)) //LeftController.Side + + + + //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) + //left trigger button can validate or delete a track + VRSelect.createAndSet(sciview.currentScene, + hmd, + listOf(OpenVRHMD.OpenVRButton.Trigger), + listOf(TrackerRole.LeftHand), + { n -> + println("the spot ${n.name} is selected") + + //delete the selected node from volume +// volume.runRecursive{it.removeChild(n)} +// eventService.publish(NodeRemovedEvent(n)) + //validate the selected node from volume + eventService.publish(NodeTaggedEvent(n)) + + }, + true) + + + VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) + + VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) + setupControllerforTracking() + } + + + + private fun setupControllerforTracking( keybindingTracking: String = "U") { + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + thread { + dumpHedgehog() + println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) + } + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true + } + } + //RightController.trigger + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true + volume.runRecursive { it.visible = true } +// playing = false + + while(true) + { + + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + + referenceTarget.visible = true + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } + + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) + } + + Thread.sleep(2) + } + } + + } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = Vector3f(center) + val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + var hedgehogsInstance = hedgehogsList.last() + val spine = hedgehogsInstance.addInstance() + spine.spatial().orientBetweenPoints(p1, p2,true,true) + spine.visible = false + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + //println("try to find intersection"); + + if(intersection is MaybeIntersects.Intersection) { + // println("got a intersection") + // get local entry and exit coordinates, and convert to UV coords + val dim = volume.getDimensions() + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + + val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) + val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) + val (samples, localDirection) = volume.sampleRay(entryUV, exitUV) ?: (null to null) + + + + if (samples != null && localDirection != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + entryUV, + exitUV, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.position, + confidence, + samples.map { it ?: 0.0f } + ) + val count = samples.filterNotNull().count { it > 0.002f } +// println("cnt: " + count.toString()) +// println(samples) + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } +// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + val hedgehogIds = AtomicInteger(0) + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog() { + //println("size of hedgehogslist: " + hedgehogsList.size.toString()) + var lastHedgehog = hedgehogsList.last() + println("lastHedgehog: ${lastHedgehog}") + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + println("do hedgehog Analysis") + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + + val parent = RichNode() + parent.name = "Track-$hedgehogId" + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + track.points.windowed(2, 1).forEach { pair -> + val element = Cylinder(3.0f, 1.0f, 5)//edgeMaster.addInstance() + val p0 = Vector3f(pair[0].first) * Vector3f(volumeDimensions) + val p1 = Vector3f(pair[1].first) * Vector3f(volumeDimensions) + + val tp = pair[0].second.timepoint + + element.spatial().orientBetweenPoints(p0, p1, rescale = true, reposition = true) + element.name = "edge" + element.metadata["Type"] = "edge" + parent.addChild(element) + + val pp = Icosphere(5.0f, 1)//nodeMaster.addInstance() + log.info("Local position: $p0 / $p1") + pp.name = "node-$tp" + pp.metadata["NodeTimepoint"] = tp + pp.metadata["NodePosition"] = p0 + pp.metadata["Type"] = "node" + pp.spatial().position = p0 + + pp.addAttribute(Grabable::class.java, Grabable()) + pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) + pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> + if (device.role == TrackerRole.LeftHand) { + pp.ifSpatial { + position = (device.velocity ?: Vector3f(0.0f)) * 5f + position + eventService.publish(NodeChangedEvent(pp)) + } + } + + })) + parent.addChild(pp) + + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product + + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + + + volume.addChild(parent) + eventService.publish(NodeAddedEvent(parent)) + + trackFileWriter.close() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + + command.run(OpenDirofTif::class.java, true, + hashMapOf( + "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), + "onlyFirst" to 10 + )) + .get() + + val argmap = HashMap() + command.run(VRHeadSetTrackingDemo::class.java, true, argmap) + .get() + } + } } \ No newline at end of file diff --git a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert index 9d166272..1f612d64 100644 --- a/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert +++ b/src/main/resources/sc/iview/commands/demo/animation/ParticleDemo.vert @@ -1,86 +1,86 @@ -#version 450 core -#extension GL_ARB_separate_shader_objects: enable - -layout(location = 0) in vec3 vertexPosition; -layout(location = 1) in vec3 vertexNormal; -layout(location = 2) in vec2 vertexTexCoord; -layout(location = 3) in mat4 iModelMatrix; -layout(location = 7) in vec4 vertexColor; - -layout(location = 0) out VertexData { - vec3 FragPosition; - vec3 Normal; - vec2 TexCoord; - vec4 Color; -} Vertex; - -layout(set = 2, binding = 0) uniform Matrices { - mat4 ModelMatrix; - mat4 NormalMatrix; - int isBillboard; -} ubo; - -struct Light { - float Linear; - float Quadratic; - float Intensity; - float Radius; - vec4 Position; - vec4 Color; -}; - -layout(set = 1, binding = 0) uniform LightParameters { - mat4 ViewMatrices[2]; - mat4 InverseViewMatrices[2]; - mat4 ProjectionMatrix; - mat4 InverseProjectionMatrix; - vec3 CamPosition; -}; - -layout(set = 0, binding = 0) uniform VRParameters { - mat4 projectionMatrices[2]; - mat4 inverseProjectionMatrices[2]; - mat4 headShift; - float IPD; - int stereoEnabled; -} vrParameters; - -layout(push_constant) uniform currentEye_t { - int eye; -} currentEye; - -void main() -{ -mat4 mv; - mat4 nMVP; - mat4 projectionMatrix; - - mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] * iModelMatrix + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye] * iModelMatrix); - projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; - - if(ubo.isBillboard == 1) { - mv[0][0] = 1.0f; - mv[0][1] = .0f; - mv[0][2] = .0f; - - mv[1][0] = .0f; - mv[1][1] = 1.0f; - mv[1][2] = .0f; - - mv[2][0] = .0f; - mv[2][1] = .0f; - mv[2][2] = 1.0f; - } - - nMVP = projectionMatrix*mv; - - mat4 normalMatrix = transpose(inverse(iModelMatrix)); - Vertex.Normal = mat3(normalMatrix) * normalize(vertexNormal); - Vertex.TexCoord = vertexTexCoord; - Vertex.FragPosition = vec3(iModelMatrix * vec4(vertexPosition, 1.0)); - Vertex.Color = vertexColor; - - gl_Position = nMVP * vec4(vertexPosition, 1.0); -} - - +#version 450 core +#extension GL_ARB_separate_shader_objects: enable + +layout(location = 0) in vec3 vertexPosition; +layout(location = 1) in vec3 vertexNormal; +layout(location = 2) in vec2 vertexTexCoord; +layout(location = 3) in mat4 iModelMatrix; +layout(location = 7) in vec4 vertexColor; + +layout(location = 0) out VertexData { + vec3 FragPosition; + vec3 Normal; + vec2 TexCoord; + vec4 Color; +} Vertex; + +layout(set = 2, binding = 0) uniform Matrices { + mat4 ModelMatrix; + mat4 NormalMatrix; + int isBillboard; +} ubo; + +struct Light { + float Linear; + float Quadratic; + float Intensity; + float Radius; + vec4 Position; + vec4 Color; +}; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +void main() +{ +mat4 mv; + mat4 nMVP; + mat4 projectionMatrix; + + mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] * iModelMatrix + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye] * iModelMatrix); + projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; + + if(ubo.isBillboard == 1) { + mv[0][0] = 1.0f; + mv[0][1] = .0f; + mv[0][2] = .0f; + + mv[1][0] = .0f; + mv[1][1] = 1.0f; + mv[1][2] = .0f; + + mv[2][0] = .0f; + mv[2][1] = .0f; + mv[2][2] = 1.0f; + } + + nMVP = projectionMatrix*mv; + + mat4 normalMatrix = transpose(inverse(iModelMatrix)); + Vertex.Normal = mat3(normalMatrix) * normalize(vertexNormal); + Vertex.TexCoord = vertexTexCoord; + Vertex.FragPosition = vec3(iModelMatrix * vec4(vertexPosition, 1.0)); + Vertex.Color = vertexColor; + + gl_Position = nMVP * vec4(vertexPosition, 1.0); +} + + From a7057b0af2e35ee3d300cd46eb07554b25713f5c Mon Sep 17 00:00:00 2001 From: ruoshan Date: Tue, 15 Nov 2022 15:16:08 +0100 Subject: [PATCH 16/55] Gradle: Add missing transitive dependencies --- build.gradle.kts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 9f862109..03012db4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -77,10 +77,20 @@ dependencies { implementation("net.imglib2:imglib2") implementation("net.imglib2:imglib2-roi") + implementation("org.janelia.saalfeldlab:n5") + implementation("org.janelia.saalfeldlab:n5-imglib2") + + implementation("org.apache.logging.log4j:log4j-api:2.19.0") + implementation("org.apache.logging.log4j:log4j-1.2-api:2.19.0") + implementation("org.apache.logging.log4j:log4j-core:2.19.0") + // Math dependencies // implementation(commons.math3) // implementation(misc.joml) + implementation("net.java.dev.jna:jna:5.12.0") + implementation("net.java.dev.jna:jna-platform:5.12.0") + // Kotlin dependencies implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$ktVersion") implementation("org.jetbrains.kotlin:kotlin-stdlib:$ktVersion") From 12cd940d7972ed562d10c2c5e45d38e9350f961d Mon Sep 17 00:00:00 2001 From: ruoshan Date: Tue, 15 Nov 2022 15:16:33 +0100 Subject: [PATCH 17/55] SciView: Revert failed merge for addVolume functions --- src/main/kotlin/sc/iview/SciView.kt | 219 ++++++++++------------------ 1 file changed, 76 insertions(+), 143 deletions(-) diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 535d5309..43f52d4d 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -872,34 +872,35 @@ class SciView : SceneryBase, CalibratedRealInterval { return } val data = io.open(source) - if (data is Mesh) - addMesh(data) - else if (data is graphics.scenery.Mesh) - addMesh(data) - else if (data is PointCloud) - addPointCloud(data) - else if (data is Dataset) - addVolume(data) - else if (data is RandomAccessibleInterval<*>) - addVolume(data as RandomAccessibleInterval>, source) - else if (data is List<*>) { - val list = data - require(!list.isEmpty()) { "Data source '$source' appears empty." } - val element = list[0] - if (element is RealLocalizable) { - // NB: For now, we assume all elements will be RealLocalizable. - // Highly likely to be the case, barring antagonistic importers. - val points = list as List - addPointCloud(points, source) - } else { - val type = if (element == null) "" else element.javaClass.name + when (data) { + is Mesh -> addMesh(data) + is PointCloud -> addPointCloud(data) + is graphics.scenery.Mesh -> addMesh(data) + is Dataset -> addVolume(data, floatArrayOf(1.0f, 1.0f, 1.0f)) +// is RandomAccessibleInterval<*> -> { +// val t = data.randomAccess().get() +// addVolume(data, source, floatArrayOf(1.0f, 1.0f, 1.0f)) +// } + is List<*> -> { + val list = data + require(!list.isEmpty()) { "Data source '$source' appears empty." } + val element = list[0] + if (element is RealLocalizable) { + // NB: For now, we assume all elements will be RealLocalizable. + // Highly likely to be the case, barring antagonistic importers. + val points = list as List + addPointCloud(points, source) + } else { + val type = if (element == null) "" else element.javaClass.name + throw IllegalArgumentException("Data source '" + source + // + "' contains elements of unknown type '" + type + "'") + } + } + else -> { + val type = if (data == null) "" else data.javaClass.name throw IllegalArgumentException("Data source '" + source + // - "' contains elements of unknown type '" + type + "'") + "' contains data of unknown type '" + type + "'") } - } else { - val type = if (data == null) "" else data.javaClass.name - throw IllegalArgumentException("Data source '" + source + // - "' contains data of unknown type '" + type + "'") } } @@ -1271,110 +1272,6 @@ class SciView : SceneryBase, CalibratedRealInterval { scene.addChild(node) } - /** - * Add a Dataset to the scene as a volume. Voxel resolution and name are extracted from the Dataset itself - * @param image image to add as a volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - fun addVolume(image: Dataset, block: Volume.() -> Unit = {}): Volume { - val voxelDims = FloatArray(image.numDimensions()) - - for (d in voxelDims.indices) { - val inValue = image.axis(d).averageScale(0.0, 1.0) - if (image.axis(d).unit() == null) { - voxelDims[d] = inValue.toFloat() - } else { - val imageAxisUnit = image.axis(d).unit().replace("µ", "u") - val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") - - voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() - } - } - - logger.info("Adding with ${voxelDims.joinToString(",")}") - val v = addVolume(image, voxelDims, block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a Dataset as a Volume with the specified voxel dimensions - * @param image image to add as a volume - * @param voxelDimensions dimensions of voxels in volume - * @return a Node corresponding to the Volume - */ - @JvmOverloads - @Suppress("UNCHECKED_CAST") - fun addVolume(image: Dataset, voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image.imgPlus as RandomAccessibleInterval>, image.name ?: "Volume", - *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", block: Volume.() -> Unit = {}): Volume { - val v = addVolume(image, name, 1f, 1f, 1f, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add a RandomAccessibleInterval to the image - * @param image image to add as a volume - * @param pixel type of image - * @return a Node corresponding to the volume - */ - fun > addVolume(image: RandomAccessibleInterval, voxelDimensions: FloatArray, block: Volume.() -> Unit): Volume { - val v = addVolume(image, "volume", *voxelDimensions, block = block) - imageToVolumeMap[image] = v - return v - } - - /** - * Add an IterableInterval as a Volume - * @param image - * @param - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, "Volume") - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - - /** - * Add an IterableInterval as a Volume - * @param image image to add as a volume - * @param name name of image - * @param pixel type of image - * @return a Node corresponding to the Volume - */ - @Suppress("UNCHECKED_CAST") - @Throws(Exception::class) - fun > addVolume(image: IterableInterval, name: String = "Volume"): Volume { - return if (image is RandomAccessibleInterval<*>) { - val v = addVolume(image as RandomAccessibleInterval>, name, 1f, 1f, 1f) - imageToVolumeMap[image] = v - v - } else { - throw Exception("Unsupported Volume type:$image") - } - } - /** * Set the colormap using an ImageJ LUT name * @param n node to apply colormap to @@ -1396,7 +1293,7 @@ class SciView : SceneryBase, CalibratedRealInterval { fun setColormap(n: Node, colorTable: ColorTable) { val copies = 16 val byteBuffer = ByteBuffer.allocateDirect( - 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture + 4 * colorTable.length * copies) // Num bytes * num components * color map length * height of color map texture val tmp = ByteArray(4 * colorTable.length) for (k in 0 until colorTable.length) { for (c in 0 until colorTable.componentCount) { @@ -1420,7 +1317,7 @@ class SciView : SceneryBase, CalibratedRealInterval { } /** - * Adss a SourceAndConverter to the scene. + * Adds a SourceAndConverter to the scene. * * @param sac The SourceAndConverter to add * @param name Name of the dataset @@ -1432,17 +1329,49 @@ class SciView : SceneryBase, CalibratedRealInterval { fun > addVolume(sac: SourceAndConverter, numTimepoints: Int, name: String = "Volume", - vararg voxelDimensions: Float, + voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { val sources: MutableList> = ArrayList() sources.add(sac) - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) + val v = addVolume(sources, numTimepoints, name, voxelDimensions, block = block) imageToVolumeMap[sources] = v imageToVolumeMap[sac] = v return v } + /** + * Adds a [Dataset]-backed [Volume] to the scene. + * + * @param image The [Dataset] to add. + * @param voxelDimensions An array containing the relative voxel dimensions of the dataset. + * @param block A lambda with additional code to execute upon adding the volume. + * @return The volume node corresponding to the [Dataset]-backed volume that has just been added. + */ + fun addVolume( + image: Dataset, + voxelDimensions: FloatArray? = null, + block: Volume.() -> Unit = {} + ): Volume { + return if(voxelDimensions == null) { + val voxelDims = FloatArray(image.numDimensions()) + for (d in voxelDims.indices) { + val inValue = image.axis(d).averageScale(0.0, 1.0) + if (image.axis(d).unit() == null) { + voxelDims[d] = inValue.toFloat() + } else { + val imageAxisUnit = image.axis(d).unit().replace("µ", "u") + val sciviewAxisUnit = axis(d)!!.unit().replace("µ", "u") + + voxelDims[d] = unitService.value(inValue, imageAxisUnit, sciviewAxisUnit).toFloat() + } + } + addVolume(image.imgPlus, image.name, voxelDims, block) + } else { + addVolume(image.imgPlus, image.name, voxelDimensions, block) + } + } + /** * Add an IterableInterval to the image with the specified voxelDimensions and name * This version of addVolume does most of the work @@ -1453,8 +1382,12 @@ class SciView : SceneryBase, CalibratedRealInterval { * @return a Node corresponding to the Volume */ @JvmOverloads - fun > addVolume(image: RandomAccessibleInterval, name: String = "Volume", - vararg voxelDimensions: Float, block: Volume.() -> Unit = {}): Volume { + fun > addVolume( + image: RandomAccessibleInterval, + name: String = "Volume", + voxelDimensions: FloatArray = floatArrayOf(1.0f, 1.0f, 1.0f), + block: Volume.() -> Unit = {} + ): Volume { //log.debug( "Add Volume " + name + " image: " + image ); val dimensions = LongArray(image.numDimensions()) image.dimensions(dimensions) @@ -1479,12 +1412,13 @@ class SciView : SceneryBase, CalibratedRealInterval { s = RandomAccessibleIntervalSource(stack, voxelType, sourceTransform, name) } val source = BigDataViewer.wrapWithTransformedSource( - SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) + SourceAndConverter(s, BigDataViewer.createConverterToARGB(voxelType))) converterSetups.add(BigDataViewer.createConverterSetup(source, setupId.getAndIncrement())) sources.add(source) } - val v = addVolume(sources, numTimepoints, name, *voxelDimensions, block = block) - v.metadata.set("RandomAccessibleInterval", image) + + val v = addVolume(sources, numTimepoints, name, voxelDimensions, block = block) + v.metadata["RandomAccessibleInterval"] = image imageToVolumeMap[image] = v return v } @@ -1502,7 +1436,7 @@ class SciView : SceneryBase, CalibratedRealInterval { */ @JvmOverloads @Suppress("UNCHECKED_CAST") - fun > addVolume(sources: List>, + fun > addVolume(sources: List>, converterSetups: ArrayList, numTimepoints: Int, name: String = "Volume", @@ -1568,7 +1502,7 @@ class SciView : SceneryBase, CalibratedRealInterval { fun > addVolume(sources: List>, numTimepoints: Int, name: String = "Volume", - vararg voxelDimensions: Float, + voxelDimensions: FloatArray, block: Volume.() -> Unit = {}): Volume { var setupId = 0 val converterSetups = ArrayList() @@ -1703,7 +1637,6 @@ class SciView : SceneryBase, CalibratedRealInterval { } else { renderer!!.toggleVR() } -// renderer!!.toggleVR() } /** @@ -1883,4 +1816,4 @@ class SciView : SceneryBase, CalibratedRealInterval { return create() } } -} +} \ No newline at end of file From e909c792d26d230b4e2f038e3f8a96534ddc491e Mon Sep 17 00:00:00 2001 From: ruoshan Date: Mon, 19 Dec 2022 09:15:42 +0100 Subject: [PATCH 18/55] clean and add comment --- build.gradle.kts | 2 + src/main/kotlin/sc/iview/SciView.kt | 1 + src/main/kotlin/sc/iview/SplashLabel.kt | 2 +- .../advanced/ConfirmableClickBehaviour.kt | 2 +- .../commands/demo/advanced/EyeTrackingDemo.kt | 23 ------- .../demo/advanced/HedgehogAnalysis.kt | 68 +++---------------- .../commands/demo/advanced/SpineMetadata.kt | 5 +- .../sc/iview/commands/demo/advanced/Test.kt | 4 -- .../demo/advanced/VRControllerTrackingDemo.kt | 40 ++--------- .../demo/advanced/VRHeadSetTrackingDemo.kt | 41 ++++++----- 10 files changed, 39 insertions(+), 149 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 03012db4..4323776c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -173,6 +173,8 @@ tasks { val dependenciesNode = asNode().appendNode("dependencies") val propertiesNode = asNode().appendNode("properties") propertiesNode.appendNode("inceptionYear", 2016) + propertiesNode.appendNode("lwjgl.version", "3.3.1") + propertiesNode.appendNode("jna-platform.version", "5.12.0") // lwjgl natives lwjglNatives.forEach { nativePlatform -> diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 43f52d4d..1178890b 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -365,6 +365,7 @@ class SciView : SceneryBase, CalibratedRealInterval { * Initialization of SWING and scenery. Also triggers an initial population of lights/camera in the scene */ override fun init() { + println("hello world!") val logLevel = System.getProperty("scenery.LogLevel", "info") log.level = LogLevel.value(logLevel) LogbackUtils.setLogLevel(null, logLevel) diff --git a/src/main/kotlin/sc/iview/SplashLabel.kt b/src/main/kotlin/sc/iview/SplashLabel.kt index fde6304c..798d244d 100644 --- a/src/main/kotlin/sc/iview/SplashLabel.kt +++ b/src/main/kotlin/sc/iview/SplashLabel.kt @@ -123,7 +123,7 @@ class SplashLabel : JPanel(), ItemListener { splashImage = try { ImageIO.read(this.javaClass.getResourceAsStream("sciview-logo.png")) - } catch (e: IOException) { + } catch (e: IOException ) { logger.warn("Could not read splash image 'sciview-logo.png'") BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB) } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt index 16ce340a..58aef285 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/ConfirmableClickBehaviour.kt @@ -1,4 +1,4 @@ -package graphics.scenery.bionictracking +package sc.iview.commands.demo.advanced import org.scijava.ui.behaviour.ClickBehaviour import kotlin.concurrent.thread diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 76cbfefd..1e550091 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -1,19 +1,11 @@ package sc.iview.commands.demo.advanced -import bdv.util.BdvFunctions import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTracker import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew -import graphics.scenery.numerics.OpenSimplexNoise import graphics.scenery.numerics.Random import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects @@ -21,44 +13,29 @@ import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* -import org.scijava.Context import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin -import org.scijava.ui.UIService import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget import sc.iview.SciView import sc.iview.commands.MenuWeights import java.awt.image.DataBufferByte import java.io.BufferedWriter import java.io.ByteArrayInputStream -import java.io.File import java.io.FileWriter import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer import javax.imageio.ImageIO import kotlin.concurrent.thread import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index cf06a144..5d3e74c0 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -1,20 +1,12 @@ -package graphics.scenery.bionictracking +package sc.iview.commands.demo.advanced -import graphics.scenery.Icosphere -import graphics.scenery.Scene -import graphics.scenery.bionictracking.HedgehogAnalysis.Companion.toVector3f import org.joml.Vector3f -import org.joml.Vector4f import org.joml.Matrix4f import org.joml.Quaternionf import graphics.scenery.utils.LazyLogger import graphics.scenery.utils.extensions.* -import org.scijava.log.LogService -import org.scijava.plugin.Parameter import org.slf4j.LoggerFactory import java.io.File -import kotlin.math.abs -import kotlin.math.pow import kotlin.math.sqrt /** @@ -34,8 +26,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix private set data class Track( - val points: List>, - val confidence: Float + val points: List>, + val confidence: Float ) init { @@ -102,7 +94,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } fun Iterable.stddev() = sqrt((this.map { (it - this.average()) * (it - this.average()) }.sum() / this.count())) -// fun Iterable.avg() = (this.map { it}.sum() / this.count()) fun Vector3f.toQuaternionf(forward: Vector3f = Vector3f(0.0f, 0.0f, -1.0f)): Quaternionf { val cross = forward.cross(this) @@ -114,14 +105,13 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } fun run(): Track? { - logger.info("run track analysis") + val startingThreshold = 0.002f val localMaxThreshold = 0.001f val zscoreThreshold = 2.0f val removeTooFarThreshold = 5.0f if(timepoints.isEmpty()) { - logger.info("timepoints is empty") return null } @@ -133,8 +123,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") -// val remainingTimepoints = timepoints.entries.drop(timepoints.entries.indexOf(startingPoint)) - timepoints.filter { it.key > startingPoint.key } .forEach { timepoints.remove(it.key) } @@ -148,17 +136,12 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") if(maxIndices.isNotEmpty()) { -// filter the maxIndices which are too far away - maxIndices.filter { it.first <1200}. + maxIndices. +// filter the maxIndices which are too far away, which can be removed + filter { it.first <1200}. map { index -> -// logger.info(index.toString()) val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) -// logger.info("i: " + i) -// logger.info("position: " + position) -// logger.info("dimension: "+ dimension) -// logger.info("localToWorld: "+ localToWorld) val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() -// logger.info("world position: "+ worldPosition) SpineGraphVertex(tp.key, position, worldPosition, @@ -178,29 +161,17 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix //step3: connect localMaximal points between 2 candidate spines according to the shortest path principle // get the initial vertex, this one is assumed to always be in front, and have a local max val initial = candidates.first().filter{it.value>startingThreshold}.first() - logger.info("initial:"+initial) - logger.info("candidates number: "+ candidates.size) var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> -// System.out.logger.info("time: ${time}") -// logger.info("vs: ${vs}") val distances = vs .filter { it.value > localMaxThreshold } .map { vertex -> val t = current.worldPosition - vertex.worldPosition val distance = t.length() -// logger.info("current worldposition:"+ current.worldPosition) -// logger.info("vertex.worldposition"+vertex.worldPosition) vertex to distance } .sortedBy { it.second } - //logger.info("distances.size: "+distances.size) - //logger.info("distances.firstOrNull()?.second: "+ distances.firstOrNull()?.second) -// if(distances.firstOrNull()?.second != null && distances.firstOrNull()?.second!! > 0) -// { -// logger.info("Minimum distance for t=$time d=${distances.firstOrNull()?.second} a=${distances.firstOrNull()?.first?.index} ") -// } -// + val closest = distances.firstOrNull()?.first if(closest != null && distances.firstOrNull()?.second!! >0) { current.next = closest @@ -214,8 +185,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val beforeCount = shortestPath.size -// System.out.logger.info("before short path:"+ shortestPath.size) - var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") @@ -232,15 +201,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix it.getOrNull(2)?.previous = it.getOrNull(1) } -// logger.info("check which one is removed") -// shortestPath.forEach { -// if(it.distance() >= removeTooFarThreshold * avgPathLength) -// { -// logger.info("current index= ${it.index}, distance = ${it.distance()}, next index = ${it.next?.index}" ) -// } -// } } -// + avgPathLength = shortestPath.map { it.distance() }.average().toFloat() stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() @@ -256,9 +218,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix }.flatten() shortestPath = shortestPath.filterIndexed { index, _ -> index !in outliers }.toMutableList() - - //logger.info("Average path length=$avgPathLength, stddev=$stdDevPathLength") - remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } shortestPath.windowed(3, 1, partialWindows = true).forEach { @@ -272,8 +231,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix val afterCount = shortestPath.size logger.info("Pruned ${beforeCount - afterCount} vertices due to path length") -// logger.info("Final distances: ${shortestPath.joinToString { "d = ${it.distance()}" }}") -// logger.info(shortestPath.toString()) val singlePoints = shortestPath .groupBy { it.timepoint } .mapNotNull { vs -> vs.value.maxByOrNull{ it.metadata.confidence } } @@ -324,7 +281,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } private fun String.toVector3f(): Vector3f { -// System.out.logger.info(this) val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} if (array[0] == "+Inf" || array[0] == "-Inf") @@ -334,7 +290,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } private fun String.toQuaternionf(): Quaternionf { -// System.out.logger.info(this) val array = this.replace("(", "").replace(")", "").trim().split(" ").filterNot { it == ""} return Quaternionf(array[0].toFloat(), array[1].toFloat(), array[2].toFloat(), array[3].toFloat()) } @@ -422,13 +377,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix fun main(args: Array) { val logger = LoggerFactory.getLogger("HedgehogAnalysisMain") -// if(args.isEmpty()) { -// logger.error("Sorry, but a file name is needed.") -// return -// } val file = File("C:\\Users\\lanru\\Desktop\\BionicTracking-generated-2021-11-29 19.37.43\\Hedgehog_1_2021-11-29 19.38.32.csv") -// val analysis = HedgehogAnalysis.fromIncompleteCSV(file) val analysis = HedgehogAnalysis.fromCSV(file) val results = analysis.run() logger.info("Results: \n$results") diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt index d1ebca32..5192d096 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt @@ -1,8 +1,5 @@ -package graphics.scenery.bionictracking +package sc.iview.commands.demo.advanced -//import cleargl.GLMatrix -//import cleargl.GLVector -import com.jogamp.opengl.math.Quaternion import org.joml.Quaternionf import org.joml.Vector3f diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index 9080e735..c57150ba 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -2,13 +2,10 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* import graphics.scenery.attribute.material.Material -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.OpenVRHMD import graphics.scenery.numerics.Random import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import graphics.scenery.utils.LazyLogger import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus @@ -18,7 +15,6 @@ import graphics.scenery.volumes.Volume import org.joml.* import org.scijava.command.Command import org.scijava.command.CommandService -import org.scijava.log.LogService import org.scijava.plugin.Menu import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 3b077144..c3099ff6 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -1,65 +1,36 @@ package sc.iview.commands.demo.advanced -import bdv.util.BdvFunctions import graphics.scenery.* -import graphics.scenery.backends.Renderer -import graphics.scenery.backends.ShaderType -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.behaviours.ControllerDrag import graphics.scenery.numerics.Random -import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw -import graphics.scenery.volumes.Colormap -import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume -import net.imglib2.FinalInterval -import net.imglib2.Localizable -import net.imglib2.RandomAccessibleInterval -import net.imglib2.img.array.ArrayImgs -import net.imglib2.position.FunctionRandomAccessible -import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* -import org.scijava.Context import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin -import org.scijava.ui.UIService import org.scijava.ui.behaviour.ClickBehaviour -import org.scijava.widget.FileWidget import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte import java.io.BufferedWriter -import java.io.ByteArrayInputStream -import java.io.File import java.io.FileWriter import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.function.BiConsumer -import javax.imageio.ImageIO import kotlin.concurrent.thread -import kotlin.math.PI -import net.imglib2.img.Img -import net.imglib2.view.Views -import org.lwjgl.openvr.OpenVR import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.controls.* import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import org.scijava.ui.behaviour.DragBehaviour -import sc.iview.commands.demo.animation.ParticleDemo @Plugin(type = Command::class, menuRoot = "SciView", @@ -407,7 +378,7 @@ class VRControllerTrackingDemo: Command{ } private fun setupControllerforTracking( keybindingTracking: String = "U") { - println("setupControllerforTracking") + thread { val cam = sciview.camera as? DetachedHeadCamera ?: return@thread @@ -433,13 +404,14 @@ class VRControllerTrackingDemo: Command{ volume.runRecursive { it.visible = true } playing = true - println("test") while(true) { + /** + * the following code is added to detect right controller + */ if(!hmd.getTrackedDevices(TrackedDeviceType.Controller).containsKey("Controller-2")) { - //println("null") continue } else @@ -573,7 +545,6 @@ class VRControllerTrackingDemo: Command{ } if(track == null) { -// logger.warn("No track returned") sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) return } @@ -581,8 +552,6 @@ class VRControllerTrackingDemo: Command{ lastHedgehog.metadata["HedgehogAnalysis"] = track lastHedgehog.metadata["Spines"] = spines -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val master = Cylinder(0.1f, 1.0f, 10) master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) @@ -609,7 +578,6 @@ class VRControllerTrackingDemo: Command{ val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - //mInstanced.instances.add(element) val pp = Icosphere(0.01f, 1) pp.spatial().position = p0w pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index 3d8d6837..eb4a6342 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -1,9 +1,6 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* -import graphics.scenery.bionictracking.ConfirmableClickBehaviour -import graphics.scenery.bionictracking.HedgehogAnalysis -import graphics.scenery.bionictracking.SpineMetadata import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole @@ -39,9 +36,7 @@ import org.scijava.event.EventService import sc.iview.commands.file.OpenDirofTif import sc.iview.event.NodeAddedEvent import sc.iview.event.NodeChangedEvent -import sc.iview.event.NodeRemovedEvent import sc.iview.event.NodeTaggedEvent -import kotlin.concurrent.fixedRateTimer @Plugin(type = Command::class, menuRoot = "SciView", @@ -216,7 +211,7 @@ class VRHeadSetTrackingDemo: Command{ { val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - //LeftController.up.down.left.right + //set up move action for moving forward, back, left and right sciview.sceneryInputHandler?.let { handler -> hashMapOf( "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), @@ -231,6 +226,7 @@ class VRHeadSetTrackingDemo: Command{ } } + //hedgehog has three modes of visibility, 1. hide hedgehog, 2, show hedgehog for per timepoint, 3. show full hedgehog val toggleHedgehog = ClickBehaviour { _, _ -> val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) @@ -259,6 +255,7 @@ class VRHeadSetTrackingDemo: Command{ } } + //adjust the direction of playing volume val nextTimepoint = ClickBehaviour { _, _ -> skipToNext = true } @@ -267,6 +264,7 @@ class VRHeadSetTrackingDemo: Command{ skipToPrevious = true } + //Speeding up the playing of volume or enlarge the scale of volume depending on whether the volume is in playing val fasterOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) @@ -274,11 +272,10 @@ class VRHeadSetTrackingDemo: Command{ } else { volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) volume.spatial().scale *= Vector3f(volumeScaleFactor) -// println("volumeScaleFactor is " + volumeScaleFactor) -// println("scale is :" + volume.spatial().scale) } } + //slower the playing of volume or reduce the scale of volume depending on whether the volume is in play val slowerOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) @@ -286,11 +283,10 @@ class VRHeadSetTrackingDemo: Command{ } else { volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) volume.spatial().scale *= Vector3f(volumeScaleFactor) -// println("volumeScaleFactor is " + volumeScaleFactor) -// println("scale is :" + volume.spatial().scale) } } + //click the button to play or pause the volume val playPause = ClickBehaviour { _, _ -> playing = !playing if(playing) { @@ -300,6 +296,7 @@ class VRHeadSetTrackingDemo: Command{ } } + //delete the last hedgehog val deleteLastHedgehog = ConfirmableClickBehaviour( armedAction = { timeout -> cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, @@ -362,7 +359,7 @@ class VRHeadSetTrackingDemo: Command{ //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) - //left trigger button can validate or delete a track + //left trigger button can validate or delete a track, the function should be arranged to two different button in the future VRSelect.createAndSet(sciview.currentScene, hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), @@ -370,10 +367,16 @@ class VRHeadSetTrackingDemo: Command{ { n -> println("the spot ${n.name} is selected") - //delete the selected node from volume + /** + * delete the selected node from volume + **/ // volume.runRecursive{it.removeChild(n)} // eventService.publish(NodeRemovedEvent(n)) - //validate the selected node from volume + + + /* + validate the selected node from volume, the tag event is designed specially for tag of Elephant + */ eventService.publish(NodeTaggedEvent(n)) }, @@ -414,6 +417,7 @@ class VRHeadSetTrackingDemo: Command{ volume.visible = true volume.runRecursive { it.visible = true } + // playing = false while(true) @@ -455,11 +459,8 @@ class VRHeadSetTrackingDemo: Command{ //println("try to find intersection"); if(intersection is MaybeIntersects.Intersection) { - // println("got a intersection") // get local entry and exit coordinates, and convert to UV coords val dim = volume.getDimensions() - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) @@ -482,12 +483,8 @@ class VRHeadSetTrackingDemo: Command{ confidence, samples.map { it ?: 0.0f } ) - val count = samples.filterNotNull().count { it > 0.002f } -// println("cnt: " + count.toString()) -// println(samples) spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } } } } @@ -535,6 +532,7 @@ class VRHeadSetTrackingDemo: Command{ h.run() } + //check whether track is null, if it is null, then let the camera show "No track returned", otherwise do analysis if(track == null) { // logger.warn("No track returned") sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) @@ -575,6 +573,7 @@ class VRHeadSetTrackingDemo: Command{ pp.metadata["Type"] = "node" pp.spatial().position = p0 + //give attributes to these nodes to make them grable, touchable and selectable, for more detailed usage check VRControllerExample in scenery pp.addAttribute(Grabable::class.java, Grabable()) pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> @@ -601,7 +600,7 @@ class VRHeadSetTrackingDemo: Command{ } companion object { - + //run function from here, it will automatically choose the volume for rendering, please give the correct location of volume @Throws(Exception::class) @JvmStatic fun main(args: Array) { From 3735f6ccfd84c82ec1c2872d0bf70fd24a255e2c Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 4 Jul 2023 11:21:20 +0200 Subject: [PATCH 19/55] Gradle Build: update to scenery 0.9.0 --- build.gradle.kts | 41 ++++++++++++++++++++--------- src/main/kotlin/sc/iview/SciView.kt | 10 +++---- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e28b9418..b0ab7430 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,14 +19,18 @@ plugins { signing } +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + repositories { mavenCentral() maven("https://maven.scijava.org/content/groups/public") - maven("https://jitpack.io") } dependencies { - val ktVersion = "1.7.20" + val ktVersion = "1.8.20" implementation(platform("org.scijava:pom-scijava:31.1.0")) // Graphics dependencies @@ -36,7 +40,7 @@ dependencies { exclude("org.lwjgl") } - val sceneryVersion = "0a32128" + val sceneryVersion = "0.8.0" api("graphics.scenery:scenery:$sceneryVersion") { version { strictly(sceneryVersion) } exclude("org.biojava.thirdparty", "forester") @@ -47,6 +51,12 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.4") implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4") implementation("org.msgpack:jackson-dataformat-msgpack:0.9.3") + implementation("net.java.dev.jna:jna-platform:5.11.0") + implementation("net.clearvolume:cleargl") + implementation("org.janelia.saalfeldlab:n5") + implementation("org.janelia.saalfeldlab:n5-imglib2") + implementation("org.apache.logging.log4j:log4j-api:2.20.0") + implementation("org.apache.logging.log4j:log4j-1.2-api:2.20.0") implementation("com.formdev:flatlaf:2.6") @@ -77,6 +87,9 @@ dependencies { implementation("net.imglib2:imglib2") implementation("net.imglib2:imglib2-roi") + // XDG support + implementation("dev.dirs:directories:26") + // Math dependencies // implementation(commons.math3) // implementation(misc.joml) @@ -102,6 +115,7 @@ dependencies { implementation(platform(kotlin("bom"))) implementation(kotlin("stdlib-jdk8")) testImplementation(kotlin("test-junit")) + testImplementation("org.slf4j:slf4j-simple:1.7.36") implementation("sc.fiji:bigdataviewer-core") implementation("sc.fiji:bigdataviewer-vistools") @@ -122,7 +136,7 @@ dependencies { tasks { withType().all { val version = System.getProperty("java.version").substringBefore('.').toInt() - val default = if (version == 1) "1.8" else "$version" + val default = if (version == 1) "11" else "$version" kotlinOptions { jvmTarget = project.properties["jvmTarget"]?.toString() ?: default freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") @@ -151,9 +165,6 @@ tasks { parent.appendNode("relativePath") val repositories = asNode().appendNode("repositories") - val jitpackRepo = repositories.appendNode("repository") - jitpackRepo.appendNode("id", "jitpack.io") - jitpackRepo.appendNode("url", "https://jitpack.io") val scijavaRepo = repositories.appendNode("repository") scijavaRepo.appendNode("id", "scijava.public") @@ -204,6 +215,9 @@ tasks { // add jvrpn property because it only has runtime native deps propertiesNode.appendNode("jvrpn.version", "1.2.0") + // add correct lwjgl version + propertiesNode.appendNode("lwjgl.version", "3.3.1") + val versionedArtifacts = listOf("scenery", "flatlaf", "kotlin-stdlib-common", @@ -216,6 +230,7 @@ tasks { "jackson-dataformat-yaml", "jackson-dataformat-msgpack", "jogl-all", + "jna-platform", "kotlin-bom", "lwjgl", "lwjgl-glfw", @@ -230,7 +245,7 @@ tasks { val toSkip = listOf("pom-scijava") - configurations.implementation.allDependencies.forEach { + configurations.implementation.get().allDependencies.forEach { val artifactId = it.name if (!toSkip.contains(artifactId)) { @@ -310,7 +325,7 @@ tasks { register("runMain", JavaExec::class.java) { classpath = sourceSets.main.get().runtimeClasspath - main = "sc.iview.Main" + mainClass.set("sc.iview.Main") val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } @@ -325,7 +340,7 @@ tasks { register("runImageJMain", JavaExec::class.java) { classpath = sourceSets.main.get().runtimeClasspath - main = "sc.iview.ImageJMain" + mainClass.set("sc.iview.ImageJMain") val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } @@ -355,7 +370,7 @@ tasks { println("Registering $exampleName of $exampleType") register(name = className.substringAfterLast(".")) { classpath = sourceSets.test.get().runtimeClasspath - main = className + mainClass.set(className) group = "demos.$exampleType" val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } @@ -382,7 +397,7 @@ tasks { // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") // } - main = "$target" + mainClass.set("$target") val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } val additionalArgs = System.getenv("SCENERY_JVM_ARGS") @@ -392,7 +407,7 @@ tasks { allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } } - println("Will run target $target with classpath $classpath, main=$main") + println("Will run target $target with classpath $classpath, main=${mainClass.get()}") println("JVM arguments passed to target: $allJvmArgs") } } diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 88eee9b4..c7da116f 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -37,11 +37,11 @@ import bdv.util.RandomAccessibleIntervalSource4D import bdv.util.volatiles.VolatileView import bdv.viewer.Source import bdv.viewer.SourceAndConverter +import bvv.core.VolumeViewerOptions import dev.dirs.ProjectDirectories import graphics.scenery.* import graphics.scenery.Scene.RaycastResult import graphics.scenery.backends.Renderer -import graphics.scenery.backends.opengl.OpenGLRenderer import graphics.scenery.backends.vulkan.VulkanRenderer import graphics.scenery.controls.InputHandler import graphics.scenery.controls.OpenVRHMD @@ -110,7 +110,6 @@ import sc.iview.ui.CustomPropertyUI import sc.iview.ui.MainWindow import sc.iview.ui.SwingMainWindow import sc.iview.ui.TaskManager -import tpietzsch.example2.VolumeViewerOptions import java.awt.event.WindowListener import java.io.IOException import java.net.URL @@ -388,10 +387,9 @@ class SciView : SceneryBase, CalibratedRealInterval { versionString = versionString.substring(0, 5) val launcherVersion = Version(versionString) val nonWorkingVersion = Version("4.0.5") - if (launcherVersion.compareTo(nonWorkingVersion) <= 0 + if (launcherVersion <= nonWorkingVersion && !java.lang.Boolean.parseBoolean(System.getProperty("sciview.DisableLauncherVersionCheck", "false"))) { - logger.info("imagej-launcher version smaller or equal to non-working version ($versionString vs. 4.0.5), disabling Vulkan as rendering backend. Disable check by setting 'scenery.DisableLauncherVersionCheck' system property to 'true'.") - System.setProperty("scenery.Renderer", "OpenGLRenderer") + throw IllegalStateException("imagej-launcher version is outdated, please update your Fiji installation.") } else { logger.info("imagej-launcher version bigger that non-working version ($versionString vs. 4.0.5), all good.") } @@ -871,7 +869,7 @@ class SciView : SceneryBase, CalibratedRealInterval { } /** - * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true. + * Add Node n to the scene and set it as the active node/publish it to the event service if activePublish is true * @param n node to add to scene * @param activePublish flag to specify whether the node becomes active *and* is published in the inspector/services * @param block an optional code that will be executed as a part of adding the node From 1ecaf0ff5f132d38343382d180fbe47d08709f8e Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Thu, 6 Jul 2023 11:40:58 +0200 Subject: [PATCH 20/55] Gradle Build: fixing outdated code --- build.gradle.kts | 2 +- src/main/java/sc/iview/io/N5IO.java | 2 +- .../sc/iview/commands/demo/advanced/HedgehogAnalysis.kt | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b0ab7430..b45a92de 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -40,7 +40,7 @@ dependencies { exclude("org.lwjgl") } - val sceneryVersion = "0.8.0" + val sceneryVersion = "0.8.2-SNAPSHOT" api("graphics.scenery:scenery:$sceneryVersion") { version { strictly(sceneryVersion) } exclude("org.biojava.thirdparty", "forester") diff --git a/src/main/java/sc/iview/io/N5IO.java b/src/main/java/sc/iview/io/N5IO.java index e7f44569..1c8407c2 100644 --- a/src/main/java/sc/iview/io/N5IO.java +++ b/src/main/java/sc/iview/io/N5IO.java @@ -29,6 +29,7 @@ package sc.iview.io; import bdv.util.AxisOrder; +import bvv.core.VolumeViewerOptions; import graphics.scenery.Group; import graphics.scenery.Node; import graphics.scenery.primitives.PointCloud; @@ -46,7 +47,6 @@ import sc.iview.SciView; import sc.iview.SciViewService; import sc.iview.process.MeshConverter; -import tpietzsch.example2.VolumeViewerOptions; import java.io.File; import java.io.IOException; diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 5d3e74c0..471b7684 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -3,8 +3,8 @@ package sc.iview.commands.demo.advanced import org.joml.Vector3f import org.joml.Matrix4f import org.joml.Quaternionf -import graphics.scenery.utils.LazyLogger import graphics.scenery.utils.extensions.* +import graphics.scenery.utils.lazyLogger import org.slf4j.LoggerFactory import java.io.File import kotlin.math.sqrt @@ -16,7 +16,7 @@ import kotlin.math.sqrt */ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix4f) { - private val logger by LazyLogger() + private val logger by lazyLogger() val timepoints = LinkedHashMap>() @@ -246,7 +246,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } companion object { - private val logger by LazyLogger(System.getProperty("scenery.LogLevel", "info")) + private val logger by lazyLogger(System.getProperty("scenery.LogLevel", "info")) fun fromIncompleteCSV(csv: File, separator: String = ","): HedgehogAnalysis { logger.info("Loading spines from incomplete CSV at ${csv.absolutePath}") From 0a78f6689cc1ea2c2067f8db2bbcc3964aa6d510 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Thu, 6 Jul 2023 13:33:19 +0200 Subject: [PATCH 21/55] SciView: bring back openDirTiff and openTrackFile --- src/main/kotlin/sc/iview/SciView.kt | 132 ++++++++++++++++++ .../commands/demo/advanced/EyeTrackingDemo.kt | 6 +- 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index c7da116f..32e2d86a 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -57,6 +57,7 @@ import graphics.scenery.utils.Statistics import graphics.scenery.utils.extensions.times import graphics.scenery.volumes.Colormap import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume import graphics.scenery.volumes.Volume.Companion.fromXML import graphics.scenery.volumes.Volume.Companion.setupId @@ -86,6 +87,7 @@ import net.imglib2.type.numeric.integer.UnsignedByteType import net.imglib2.view.Views import org.joml.Quaternionf import org.joml.Vector3f +import org.joml.Vector4f import org.scijava.Context import org.scijava.`object`.ObjectService import org.scijava.display.Display @@ -101,6 +103,7 @@ import org.scijava.thread.ThreadService import org.scijava.util.ColorRGB import org.scijava.util.Colors import org.scijava.util.VersionUtils +import sc.iview.commands.demo.animation.ParticleDemo import sc.iview.event.NodeActivatedEvent import sc.iview.event.NodeAddedEvent import sc.iview.event.NodeChangedEvent @@ -111,10 +114,12 @@ import sc.iview.ui.MainWindow import sc.iview.ui.SwingMainWindow import sc.iview.ui.TaskManager import java.awt.event.WindowListener +import java.io.File import java.io.IOException import java.net.URL import java.nio.ByteBuffer import java.nio.FloatBuffer +import java.nio.file.Path import java.time.LocalDate import java.time.format.DateTimeFormatter import java.util.* @@ -129,6 +134,7 @@ import kotlin.collections.LinkedHashMap import javax.swing.JOptionPane import kotlin.math.cos import kotlin.math.sin +import kotlin.system.measureTimeMillis /** * Main SciView class. @@ -755,6 +761,132 @@ class SciView : SceneryBase, CalibratedRealInterval { } } + @Throws(IOException::class) + fun openDirTiff(source: Path, onlyFirst: Int? = null) + { + val v = Volume.fromPath(source, hub, onlyFirst) + v.name = "volume" + v.spatial().position = Vector3f(-3.0f, 10.0f, 0.0f) + v.colormap = Colormap.get("jet") + v.spatial().scale = Vector3f(15.0f, 15.0f,45.0f) + v.transferFunction = TransferFunction.ramp(0.05f, 0.8f) + v.metadata["animating"] = true + v.converterSetups.firstOrNull()?.setDisplayRange(0.0, 1500.0) + v.visible = true + + v.spatial().wantsComposeModel = true + v.spatial().updateWorld(true) +// System.out.println("v.model: " + v.model) + addChild(v) +// System.out.println("v.getDimensions: "+ v.getDimensions()) +// +// System.out.println(" v.pixelToWorldRatio: "+ v.pixelToWorldRatio) +// System.out.println("v.world.matrix: " + v.spatial().world) + } + + data class PointInTrack( + val t: Int, + val loc: Vector3f, + val cellId: Long, + val parentId: Long, + val nodeScore: Float, + val edgeScore: Float + ) + + data class Track( + val track: List, + val trackId: Int + ) + + @Throws(IOException::class) + fun openTrackFile(file: File) + { + val lines = file.readLines() + var track = ArrayList() + val tracks = ArrayList() + val separator = "," + + var lastTrackId = -1 + lines.drop(1).forEach { line -> + val tokens = line.split(separator) + val t = tokens[0].toInt() + val z = tokens[1].toFloat() -2000f + val y = tokens[2].toFloat() -800f + val x = tokens[3].toFloat() -1300f + val cellId = tokens[4].toLong() + val parentId = tokens[5].toLong() + val trackId = tokens[6].toInt() + val nodeScore = tokens[7].toFloat() + val edgeScore = tokens[8].toFloat()/45.0f + + val currentPointInTrack = PointInTrack( + t, + Vector3f(x,y,z), + cellId, + parentId, + nodeScore, + edgeScore + ) + if(lastTrackId != trackId) + { + lastTrackId = trackId + val sortedTrack = track.sortedBy { it.t } + tracks.add(Track(sortedTrack, trackId)) + + track.clear() + } + track.add(currentPointInTrack) + } + val timeCost = measureTimeMillis { + addTracks(tracks) + } + println("time: $timeCost") + } + + fun addTracks(tracks: ArrayList) + { + val rng = Random(17) + for(track in tracks) + { + if(track.trackId > 10) + { + continue + } + System.out.println("add track: "+ track.trackId.toString() ) + val master = Cylinder(0.1f, 1.0f, 10) +// master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) + master.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + master.ifMaterial{ + ambient = Vector3f(0.1f, 0f, 0f) + diffuse = Vector3f(0.05f, 0f, 0f) + metallic = 0.01f + roughness = 0.5f + } + + val mInstanced = InstancedNode(master) + mInstanced.name = "TrackID-${track.trackId}" + mInstanced.instancedProperties["Color"] = { Vector4f(1.0f) } + addNode(mInstanced) + + var cnt = 0 + val a = rng.nextFloat() + val b = rng.nextFloat() + track.track.windowed(2,1).forEach { pair -> + cnt = cnt + 1 + val element = mInstanced.addInstance() + element.name ="EdgeID-$cnt" + element.instancedProperties["Color"] = { Vector4f( a,b,pair[0].edgeScore, 1.0f) } + element.spatial().orientBetweenPoints(Vector3f(pair[0].loc).mul(0.1f) , Vector3f(pair[1].loc).mul(0.1f) , rescale = true, reposition = true) + //mInstanced.instances.add(element) + + } + } + + } + + + + /** * Open a file specified by the source path. The file can be anything that SciView knows about: mesh, volume, point cloud * @param source string of a data source diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 1e550091..499026c8 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -5,7 +5,7 @@ import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.controls.eyetracking.PupilEyeTrackerNew +import graphics.scenery.controls.eyetracking.PupilEyeTracker import graphics.scenery.numerics.Random import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects @@ -53,7 +53,7 @@ class EyeTrackingDemo: Command{ @Parameter private lateinit var log: LogService - val pupilTracker = PupilEyeTrackerNew(calibrationType = PupilEyeTrackerNew.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) lateinit var hmd: OpenVRHMD val referenceTarget = Icosphere(0.004f, 2) val calibrationTarget = Icosphere(0.02f, 2) @@ -468,7 +468,7 @@ class EyeTrackingDemo: Command{ pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { //NEW - PupilEyeTrackerNew.CalibrationType.WorldSpace -> { gaze -> + PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> if (gaze.confidence > confidenceThreshold) { val p = gaze.gazePoint() referenceTarget.visible = true From 9efcb087e198cfbbafde6c53b7305765de25f281 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:50:04 +0200 Subject: [PATCH 22/55] Gradle: fix directories dep issue --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index b45a92de..4fe787ba 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -219,6 +219,7 @@ tasks { propertiesNode.appendNode("lwjgl.version", "3.3.1") val versionedArtifacts = listOf("scenery", + "directories", "flatlaf", "kotlin-stdlib-common", "kotlin-stdlib", From 80aeccdb2a6fd79fd6c577769abfc27b9f55e5df Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:05:55 +0100 Subject: [PATCH 23/55] build: update gradle and make the build compatible with java 21 --- build.gradle.kts | 32 +++++++++++++++------ gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 60756 -> 43453 bytes gradle/wrapper/gradle-wrapper.properties | 4 ++- gradlew | 35 ++++++++++++++--------- gradlew.bat | 21 +++++++------- 6 files changed, 60 insertions(+), 34 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4fe787ba..66c8f5f7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ import org.gradle.kotlin.dsl.implementation +import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import java.net.URL import sciview.* @@ -133,16 +134,29 @@ dependencies { // } //} +kotlin { + jvmToolchain(21) +// compilerOptions { +// jvmTarget = JvmTarget.JVM_21 +// freeCompilerArgs = listOf("-Xinline-classes", "-opt-in=kotlin.RequiresOptIn") +// } +} + +java { + targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 +} + tasks { - withType().all { - val version = System.getProperty("java.version").substringBefore('.').toInt() - val default = if (version == 1) "11" else "$version" - kotlinOptions { - jvmTarget = project.properties["jvmTarget"]?.toString() ?: default - freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") - } -// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default - } +// withType().all { +// val version = System.getProperty("java.version").substringBefore('.').toInt() +// val default = if (version == 1) "21" else "$version" +// kotlinOptions { +// jvmTarget = project.properties["jvmTarget"]?.toString() ?: default +// freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") +// } +//// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default +// } test { finalizedBy(jacocoTestReport) // report is always generated after tests run } diff --git a/gradle.properties b/gradle.properties index bedcd1be..3150c532 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=2g org.gradle.caching=true -jvmTarget=11 +jvmTarget=21 #useLocalScenery=true diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832f090a2944b7473328c07c9755baa3196..e6441136f3d4ba8a0da8d277868979cfbc8ad796 100644 GIT binary patch literal 43453 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vSTxF-Vi3+ZOI=Thq2} zyQgjYY1_7^ZQHh{?P))4+qUiQJLi1&{yE>h?~jU%tjdV0h|FENbM3X(KnJdPKc?~k zh=^Ixv*+smUll!DTWH!jrV*wSh*(mx0o6}1@JExzF(#9FXgmTXVoU+>kDe68N)dkQ zH#_98Zv$}lQwjKL@yBd;U(UD0UCl322=pav<=6g>03{O_3oKTq;9bLFX1ia*lw;#K zOiYDcBJf)82->83N_Y(J7Kr_3lE)hAu;)Q(nUVydv+l+nQ$?|%MWTy`t>{havFSQloHwiIkGK9YZ79^9?AZo0ZyQlVR#}lF%dn5n%xYksXf8gnBm=wO7g_^! zauQ-bH1Dc@3ItZ-9D_*pH}p!IG7j8A_o94#~>$LR|TFq zZ-b00*nuw|-5C2lJDCw&8p5N~Z1J&TrcyErds&!l3$eSz%`(*izc;-?HAFD9AHb-| z>)id`QCrzRws^9(#&=pIx9OEf2rmlob8sK&xPCWS+nD~qzU|qG6KwA{zbikcfQrdH z+ zQg>O<`K4L8rN7`GJB0*3<3`z({lWe#K!4AZLsI{%z#ja^OpfjU{!{)x0ZH~RB0W5X zTwN^w=|nA!4PEU2=LR05x~}|B&ZP?#pNgDMwD*ajI6oJqv!L81gu=KpqH22avXf0w zX3HjbCI!n9>l046)5rr5&v5ja!xkKK42zmqHzPx$9Nn_MZk`gLeSLgC=LFf;H1O#B zn=8|^1iRrujHfbgA+8i<9jaXc;CQBAmQvMGQPhFec2H1knCK2x!T`e6soyrqCamX% zTQ4dX_E*8so)E*TB$*io{$c6X)~{aWfaqdTh=xEeGvOAN9H&-t5tEE-qso<+C!2>+ zskX51H-H}#X{A75wqFe-J{?o8Bx|>fTBtl&tcbdR|132Ztqu5X0i-pisB-z8n71%q%>EF}yy5?z=Ve`}hVh{Drv1YWL zW=%ug_&chF11gDv3D6B)Tz5g54H0mDHNjuKZ+)CKFk4Z|$RD zfRuKLW`1B>B?*RUfVd0+u8h3r-{@fZ{k)c!93t1b0+Q9vOaRnEn1*IL>5Z4E4dZ!7 ztp4GP-^1d>8~LMeb}bW!(aAnB1tM_*la=Xx)q(I0Y@__Zd$!KYb8T2VBRw%e$iSdZ zkwdMwd}eV9q*;YvrBFTv1>1+}{H!JK2M*C|TNe$ZSA>UHKk);wz$(F$rXVc|sI^lD zV^?_J!3cLM;GJuBMbftbaRUs$;F}HDEDtIeHQ)^EJJ1F9FKJTGH<(Jj`phE6OuvE) zqK^K`;3S{Y#1M@8yRQwH`?kHMq4tHX#rJ>5lY3DM#o@or4&^_xtBC(|JpGTfrbGkA z2Tu+AyT^pHannww!4^!$5?@5v`LYy~T`qs7SYt$JgrY(w%C+IWA;ZkwEF)u5sDvOK zGk;G>Mh&elvXDcV69J_h02l&O;!{$({fng9Rlc3ID#tmB^FIG^w{HLUpF+iB`|
NnX)EH+Nua)3Y(c z&{(nX_ht=QbJ%DzAya}!&uNu!4V0xI)QE$SY__m)SAKcN0P(&JcoK*Lxr@P zY&P=}&B3*UWNlc|&$Oh{BEqwK2+N2U$4WB7Fd|aIal`FGANUa9E-O)!gV`((ZGCc$ zBJA|FFrlg~9OBp#f7aHodCe{6= zay$6vN~zj1ddMZ9gQ4p32(7wD?(dE>KA2;SOzXRmPBiBc6g`eOsy+pVcHu=;Yd8@{ zSGgXf@%sKKQz~;!J;|2fC@emm#^_rnO0esEn^QxXgJYd`#FPWOUU5b;9eMAF zZhfiZb|gk8aJIw*YLp4!*(=3l8Cp{(%p?ho22*vN9+5NLV0TTazNY$B5L6UKUrd$n zjbX%#m7&F#U?QNOBXkiiWB*_tk+H?N3`vg;1F-I+83{M2!8<^nydGr5XX}tC!10&e z7D36bLaB56WrjL&HiiMVtpff|K%|*{t*ltt^5ood{FOG0<>k&1h95qPio)2`eL${YAGIx(b4VN*~nKn6E~SIQUuRH zQ+5zP6jfnP$S0iJ@~t!Ai3o`X7biohli;E zT#yXyl{bojG@-TGZzpdVDXhbmF%F9+-^YSIv|MT1l3j zrxOFq>gd2%U}?6}8mIj?M zc077Zc9fq(-)4+gXv?Az26IO6eV`RAJz8e3)SC7~>%rlzDwySVx*q$ygTR5kW2ds- z!HBgcq0KON9*8Ff$X0wOq$`T7ml(@TF)VeoF}x1OttjuVHn3~sHrMB++}f7f9H%@f z=|kP_?#+fve@{0MlbkC9tyvQ_R?lRdRJ@$qcB(8*jyMyeME5ns6ypVI1Xm*Zr{DuS zZ!1)rQfa89c~;l~VkCiHI|PCBd`S*2RLNQM8!g9L6?n`^evQNEwfO@&JJRme+uopQX0%Jo zgd5G&#&{nX{o?TQwQvF1<^Cg3?2co;_06=~Hcb6~4XWpNFL!WU{+CK;>gH%|BLOh7@!hsa(>pNDAmpcuVO-?;Bic17R}^|6@8DahH)G z!EmhsfunLL|3b=M0MeK2vqZ|OqUqS8npxwge$w-4pFVXFq$_EKrZY?BuP@Az@(k`L z`ViQBSk`y+YwRT;&W| z2e3UfkCo^uTA4}Qmmtqs+nk#gNr2W4 zTH%hhErhB)pkXR{B!q5P3-OM+M;qu~f>}IjtF%>w{~K-0*jPVLl?Chz&zIdxp}bjx zStp&Iufr58FTQ36AHU)0+CmvaOpKF;W@sMTFpJ`j;3d)J_$tNQI^c<^1o<49Z(~K> z;EZTBaVT%14(bFw2ob@?JLQ2@(1pCdg3S%E4*dJ}dA*v}_a4_P(a`cHnBFJxNobAv zf&Zl-Yt*lhn-wjZsq<9v-IsXxAxMZ58C@e0!rzhJ+D@9^3~?~yllY^s$?&oNwyH!#~6x4gUrfxplCvK#!f z$viuszW>MFEcFL?>ux*((!L$;R?xc*myjRIjgnQX79@UPD$6Dz0jutM@7h_pq z0Zr)#O<^y_K6jfY^X%A-ip>P%3saX{!v;fxT-*0C_j4=UMH+Xth(XVkVGiiKE#f)q z%Jp=JT)uy{&}Iq2E*xr4YsJ5>w^=#-mRZ4vPXpI6q~1aFwi+lQcimO45V-JXP;>(Q zo={U`{=_JF`EQj87Wf}{Qy35s8r1*9Mxg({CvOt}?Vh9d&(}iI-quvs-rm~P;eRA@ zG5?1HO}puruc@S{YNAF3vmUc2B4!k*yi))<5BQmvd3tr}cIs#9)*AX>t`=~{f#Uz0 z0&Nk!7sSZwJe}=)-R^$0{yeS!V`Dh7w{w5rZ9ir!Z7Cd7dwZcK;BT#V0bzTt>;@Cl z#|#A!-IL6CZ@eHH!CG>OO8!%G8&8t4)Ro@}USB*k>oEUo0LsljsJ-%5Mo^MJF2I8- z#v7a5VdJ-Cd%(a+y6QwTmi+?f8Nxtm{g-+WGL>t;s#epv7ug>inqimZCVm!uT5Pf6 ziEgQt7^%xJf#!aPWbuC_3Nxfb&CFbQy!(8ANpkWLI4oSnH?Q3f?0k1t$3d+lkQs{~(>06l&v|MpcFsyAv zin6N!-;pggosR*vV=DO(#+}4ps|5$`udE%Kdmp?G7B#y%H`R|i8skKOd9Xzx8xgR$>Zo2R2Ytktq^w#ul4uicxW#{ zFjG_RNlBroV_n;a7U(KIpcp*{M~e~@>Q#Av90Jc5v%0c>egEdY4v3%|K1XvB{O_8G zkTWLC>OZKf;XguMH2-Pw{BKbFzaY;4v2seZV0>^7Q~d4O=AwaPhP3h|!hw5aqOtT@ z!SNz}$of**Bl3TK209@F=Tn1+mgZa8yh(Png%Zd6Mt}^NSjy)etQrF zme*llAW=N_8R*O~d2!apJnF%(JcN??=`$qs3Y+~xs>L9x`0^NIn!8mMRFA_tg`etw z3k{9JAjnl@ygIiJcNHTy02GMAvBVqEss&t2<2mnw!; zU`J)0>lWiqVqo|ex7!+@0i>B~BSU1A_0w#Ee+2pJx0BFiZ7RDHEvE*ptc9md(B{&+ zKE>TM)+Pd>HEmdJao7U@S>nL(qq*A)#eLOuIfAS@j`_sK0UEY6OAJJ-kOrHG zjHx`g!9j*_jRcJ%>CE9K2MVf?BUZKFHY?EpV6ai7sET-tqk=nDFh-(65rhjtlKEY% z@G&cQ<5BKatfdA1FKuB=i>CCC5(|9TMW%K~GbA4}80I5%B}(gck#Wlq@$nO3%@QP_ z8nvPkJFa|znk>V92cA!K1rKtr)skHEJD;k8P|R8RkCq1Rh^&}Evwa4BUJz2f!2=MH zo4j8Y$YL2313}H~F7@J7mh>u%556Hw0VUOz-Un@ZASCL)y8}4XXS`t1AC*^>PLwIc zUQok5PFS=*#)Z!3JZN&eZ6ZDP^-c@StY*t20JhCnbMxXf=LK#;`4KHEqMZ-Ly9KsS zI2VUJGY&PmdbM+iT)zek)#Qc#_i4uH43 z@T5SZBrhNCiK~~esjsO9!qBpaWK<`>!-`b71Y5ReXQ4AJU~T2Njri1CEp5oKw;Lnm)-Y@Z3sEY}XIgSy%xo=uek(kAAH5MsV$V3uTUsoTzxp_rF=tx zV07vlJNKtJhCu`b}*#m&5LV4TAE&%KtHViDAdv#c^x`J7bg z&N;#I2GkF@SIGht6p-V}`!F_~lCXjl1BdTLIjD2hH$J^YFN`7f{Q?OHPFEM$65^!u zNwkelo*5+$ZT|oQ%o%;rBX$+?xhvjb)SHgNHE_yP%wYkkvXHS{Bf$OiKJ5d1gI0j< zF6N}Aq=(WDo(J{e-uOecxPD>XZ@|u-tgTR<972`q8;&ZD!cep^@B5CaqFz|oU!iFj zU0;6fQX&~15E53EW&w1s9gQQ~Zk16X%6 zjG`j0yq}4deX2?Tr(03kg>C(!7a|b9qFI?jcE^Y>-VhudI@&LI6Qa}WQ>4H_!UVyF z((cm&!3gmq@;BD#5P~0;_2qgZhtJS|>WdtjY=q zLnHH~Fm!cxw|Z?Vw8*~?I$g#9j&uvgm7vPr#&iZgPP~v~BI4jOv;*OQ?jYJtzO<^y z7-#C={r7CO810!^s(MT!@@Vz_SVU)7VBi(e1%1rvS!?PTa}Uv`J!EP3s6Y!xUgM^8 z4f!fq<3Wer_#;u!5ECZ|^c1{|q_lh3m^9|nsMR1#Qm|?4Yp5~|er2?W^7~cl;_r4WSme_o68J9p03~Hc%X#VcX!xAu%1`R!dfGJCp zV*&m47>s^%Ib0~-2f$6oSgn3jg8m%UA;ArcdcRyM5;}|r;)?a^D*lel5C`V5G=c~k zy*w_&BfySOxE!(~PI$*dwG><+-%KT5p?whOUMA*k<9*gi#T{h3DAxzAPxN&Xws8o9Cp*`PA5>d9*Z-ynV# z9yY*1WR^D8|C%I@vo+d8r^pjJ$>eo|j>XiLWvTWLl(^;JHCsoPgem6PvegHb-OTf| zvTgsHSa;BkbG=(NgPO|CZu9gUCGr$8*EoH2_Z#^BnxF0yM~t`|9ws_xZ8X8iZYqh! zAh;HXJ)3P&)Q0(&F>!LN0g#bdbis-cQxyGn9Qgh`q+~49Fqd2epikEUw9caM%V6WgP)532RMRW}8gNS%V%Hx7apSz}tn@bQy!<=lbhmAH=FsMD?leawbnP5BWM0 z5{)@EEIYMu5;u)!+HQWhQ;D3_Cm_NADNeb-f56}<{41aYq8p4=93d=-=q0Yx#knGYfXVt z+kMxlus}t2T5FEyCN~!}90O_X@@PQpuy;kuGz@bWft%diBTx?d)_xWd_-(!LmVrh**oKg!1CNF&LX4{*j|) zIvjCR0I2UUuuEXh<9}oT_zT#jOrJAHNLFT~Ilh9hGJPI1<5`C-WA{tUYlyMeoy!+U zhA#=p!u1R7DNg9u4|QfED-2TuKI}>p#2P9--z;Bbf4Op*;Q9LCbO&aL2i<0O$ByoI z!9;Ght733FC>Pz>$_mw(F`zU?`m@>gE`9_p*=7o=7av`-&ifU(^)UU`Kg3Kw`h9-1 z6`e6+im=|m2v`pN(2dE%%n8YyQz;#3Q-|x`91z?gj68cMrHl}C25|6(_dIGk*8cA3 zRHB|Nwv{@sP4W+YZM)VKI>RlB`n=Oj~Rzx~M+Khz$N$45rLn6k1nvvD^&HtsMA4`s=MmuOJID@$s8Ph4E zAmSV^+s-z8cfv~Yd(40Sh4JG#F~aB>WFoX7ykaOr3JaJ&Lb49=B8Vk-SQT9%7TYhv z?-Pprt{|=Y5ZQ1?od|A<_IJU93|l4oAfBm?3-wk{O<8ea+`}u%(kub(LFo2zFtd?4 zwpN|2mBNywv+d^y_8#<$r>*5+$wRTCygFLcrwT(qc^n&@9r+}Kd_u@Ithz(6Qb4}A zWo_HdBj#V$VE#l6pD0a=NfB0l^6W^g`vm^sta>Tly?$E&{F?TTX~DsKF~poFfmN%2 z4x`Dc{u{Lkqz&y!33;X}weD}&;7p>xiI&ZUb1H9iD25a(gI|`|;G^NwJPv=1S5e)j z;U;`?n}jnY6rA{V^ zxTd{bK)Gi^odL3l989DQlN+Zs39Xe&otGeY(b5>rlIqfc7Ap4}EC?j<{M=hlH{1+d zw|c}}yx88_xQr`{98Z!d^FNH77=u(p-L{W6RvIn40f-BldeF-YD>p6#)(Qzf)lfZj z?3wAMtPPp>vMehkT`3gToPd%|D8~4`5WK{`#+}{L{jRUMt zrFz+O$C7y8$M&E4@+p+oV5c%uYzbqd2Y%SSgYy#xh4G3hQv>V*BnuKQhBa#=oZB~w{azUB+q%bRe_R^ z>fHBilnRTUfaJ201czL8^~Ix#+qOHSO)A|xWLqOxB$dT2W~)e-r9;bm=;p;RjYahB z*1hegN(VKK+ztr~h1}YP@6cfj{e#|sS`;3tJhIJK=tVJ-*h-5y9n*&cYCSdg#EHE# zSIx=r#qOaLJoVVf6v;(okg6?*L_55atl^W(gm^yjR?$GplNP>BZsBYEf_>wM0Lc;T zhf&gpzOWNxS>m+mN92N0{;4uw`P+9^*|-1~$uXpggj4- z^SFc4`uzj2OwdEVT@}Q`(^EcQ_5(ZtXTql*yGzdS&vrS_w>~~ra|Nb5abwf}Y!uq6R5f&6g2ge~2p(%c< z@O)cz%%rr4*cRJ5f`n@lvHNk@lE1a*96Kw6lJ~B-XfJW%?&-y?;E&?1AacU@`N`!O z6}V>8^%RZ7SQnZ-z$(jsX`amu*5Fj8g!3RTRwK^`2_QHe;_2y_n|6gSaGyPmI#kA0sYV<_qOZc#-2BO%hX)f$s-Z3xlI!ub z^;3ru11DA`4heAu%}HIXo&ctujzE2!6DIGE{?Zs>2}J+p&C$rc7gJC35gxhflorvsb%sGOxpuWhF)dL_&7&Z99=5M0b~Qa;Mo!j&Ti_kXW!86N%n= zSC@6Lw>UQ__F&+&Rzv?gscwAz8IP!n63>SP)^62(HK98nGjLY2*e^OwOq`3O|C92? z;TVhZ2SK%9AGW4ZavTB9?)mUbOoF`V7S=XM;#3EUpR+^oHtdV!GK^nXzCu>tpR|89 zdD{fnvCaN^^LL%amZ^}-E+214g&^56rpdc@yv0b<3}Ys?)f|fXN4oHf$six)-@<;W&&_kj z-B}M5U*1sb4)77aR=@%I?|Wkn-QJVuA96an25;~!gq(g1@O-5VGo7y&E_srxL6ZfS z*R%$gR}dyONgju*D&?geiSj7SZ@ftyA|}(*Y4KbvU!YLsi1EDQQCnb+-cM=K1io78o!v*);o<XwjaQH%)uIP&Zm?)Nfbfn;jIr z)d#!$gOe3QHp}2NBak@yYv3m(CPKkwI|{;d=gi552u?xj9ObCU^DJFQp4t4e1tPzM zvsRIGZ6VF+{6PvqsplMZWhz10YwS={?`~O0Ec$`-!klNUYtzWA^f9m7tkEzCy<_nS z=&<(awFeZvt51>@o_~>PLs05CY)$;}Oo$VDO)?l-{CS1Co=nxjqben*O1BR>#9`0^ zkwk^k-wcLCLGh|XLjdWv0_Hg54B&OzCE^3NCP}~OajK-LuRW53CkV~Su0U>zN%yQP zH8UH#W5P3-!ToO-2k&)}nFe`t+mdqCxxAHgcifup^gKpMObbox9LFK;LP3}0dP-UW z?Zo*^nrQ6*$FtZ(>kLCc2LY*|{!dUn$^RW~m9leoF|@Jy|M5p-G~j%+P0_#orRKf8 zvuu5<*XO!B?1E}-*SY~MOa$6c%2cM+xa8}_8x*aVn~57v&W(0mqN1W`5a7*VN{SUH zXz98DDyCnX2EPl-`Lesf`=AQT%YSDb`$%;(jUTrNen$NPJrlpPDP}prI>Ml!r6bCT;mjsg@X^#&<}CGf0JtR{Ecwd&)2zuhr#nqdgHj+g2n}GK9CHuwO zk>oZxy{vcOL)$8-}L^iVfJHAGfwN$prHjYV0ju}8%jWquw>}_W6j~m<}Jf!G?~r5&Rx)!9JNX!ts#SGe2HzobV5); zpj@&`cNcO&q+%*<%D7za|?m5qlmFK$=MJ_iv{aRs+BGVrs)98BlN^nMr{V_fcl_;jkzRju+c-y?gqBC_@J0dFLq-D9@VN&-`R9U;nv$Hg?>$oe4N&Ht$V_(JR3TG^! zzJsbQbi zFE6-{#9{G{+Z}ww!ycl*7rRdmU#_&|DqPfX3CR1I{Kk;bHwF6jh0opI`UV2W{*|nn zf_Y@%wW6APb&9RrbEN=PQRBEpM(N1w`81s=(xQj6 z-eO0k9=Al|>Ej|Mw&G`%q8e$2xVz1v4DXAi8G};R$y)ww638Y=9y$ZYFDM$}vzusg zUf+~BPX>(SjA|tgaFZr_e0{)+z9i6G#lgt=F_n$d=beAt0Sa0a7>z-?vcjl3e+W}+ z1&9=|vC=$co}-Zh*%3588G?v&U7%N1Qf-wNWJ)(v`iO5KHSkC5&g7CrKu8V}uQGcfcz zmBz#Lbqwqy#Z~UzHgOQ;Q-rPxrRNvl(&u6ts4~0=KkeS;zqURz%!-ERppmd%0v>iRlEf+H$yl{_8TMJzo0 z>n)`On|7=WQdsqhXI?#V{>+~}qt-cQbokEbgwV3QvSP7&hK4R{Z{aGHVS3;+h{|Hz z6$Js}_AJr383c_+6sNR|$qu6dqHXQTc6?(XWPCVZv=)D#6_;D_8P-=zOGEN5&?~8S zl5jQ?NL$c%O)*bOohdNwGIKM#jSAC?BVY={@A#c9GmX0=T(0G}xs`-%f3r=m6-cpK z!%waekyAvm9C3%>sixdZj+I(wQlbB4wv9xKI*T13DYG^T%}zZYJ|0$Oj^YtY+d$V$ zAVudSc-)FMl|54n=N{BnZTM|!>=bhaja?o7s+v1*U$!v!qQ%`T-6fBvmdPbVmro&d zk07TOp*KuxRUSTLRrBj{mjsnF8`d}rMViY8j`jo~Hp$fkv9F_g(jUo#Arp;Xw0M$~ zRIN!B22~$kx;QYmOkos@%|5k)!QypDMVe}1M9tZfkpXKGOxvKXB!=lo`p?|R1l=tA zp(1}c6T3Fwj_CPJwVsYtgeRKg?9?}%oRq0F+r+kdB=bFUdVDRPa;E~~>2$w}>O>v=?|e>#(-Lyx?nbg=ckJ#5U6;RT zNvHhXk$P}m9wSvFyU3}=7!y?Y z=fg$PbV8d7g25&-jOcs{%}wTDKm>!Vk);&rr;O1nvO0VrU&Q?TtYVU=ir`te8SLlS zKSNmV=+vF|ATGg`4$N1uS|n??f}C_4Sz!f|4Ly8#yTW-FBfvS48Tef|-46C(wEO_%pPhUC5$-~Y?!0vFZ^Gu`x=m7X99_?C-`|h zfmMM&Y@zdfitA@KPw4Mc(YHcY1)3*1xvW9V-r4n-9ZuBpFcf{yz+SR{ zo$ZSU_|fgwF~aakGr(9Be`~A|3)B=9`$M-TWKipq-NqRDRQc}ABo*s_5kV%doIX7LRLRau_gd@Rd_aLFXGSU+U?uAqh z8qusWWcvgQ&wu{|sRXmv?sl=xc<$6AR$+cl& zFNh5q1~kffG{3lDUdvEZu5c(aAG~+64FxdlfwY^*;JSS|m~CJusvi-!$XR`6@XtY2 znDHSz7}_Bx7zGq-^5{stTRy|I@N=>*y$zz>m^}^{d&~h;0kYiq8<^Wq7Dz0w31ShO^~LUfW6rfitR0(=3;Uue`Y%y@ex#eKPOW zO~V?)M#AeHB2kovn1v=n^D?2{2jhIQd9t|_Q+c|ZFaWt+r&#yrOu-!4pXAJuxM+Cx z*H&>eZ0v8Y`t}8{TV6smOj=__gFC=eah)mZt9gwz>>W$!>b3O;Rm^Ig*POZP8Rl0f zT~o=Nu1J|lO>}xX&#P58%Yl z83`HRs5#32Qm9mdCrMlV|NKNC+Z~ z9OB8xk5HJ>gBLi+m@(pvpw)1(OaVJKs*$Ou#@Knd#bk+V@y;YXT?)4eP9E5{J%KGtYinNYJUH9PU3A}66c>Xn zZ{Bn0<;8$WCOAL$^NqTjwM?5d=RHgw3!72WRo0c;+houoUA@HWLZM;^U$&sycWrFd zE7ekt9;kb0`lps{>R(}YnXlyGY}5pPd9zBpgXeJTY_jwaJGSJQC#-KJqmh-;ad&F- z-Y)E>!&`Rz!HtCz>%yOJ|v(u7P*I$jqEY3}(Z-orn4 zlI?CYKNl`6I){#2P1h)y(6?i;^z`N3bxTV%wNvQW+eu|x=kbj~s8rhCR*0H=iGkSj zk23lr9kr|p7#qKL=UjgO`@UnvzU)`&fI>1Qs7ubq{@+lK{hH* zvl6eSb9%yngRn^T<;jG1SVa)eA>T^XX=yUS@NCKpk?ovCW1D@!=@kn;l_BrG;hOTC z6K&H{<8K#dI(A+zw-MWxS+~{g$tI7|SfP$EYKxA}LlVO^sT#Oby^grkdZ^^lA}uEF zBSj$weBJG{+Bh@Yffzsw=HyChS(dtLE3i*}Zj@~!_T-Ay7z=B)+*~3|?w`Zd)Co2t zC&4DyB!o&YgSw+fJn6`sn$e)29`kUwAc+1MND7YjV%lO;H2}fNy>hD#=gT ze+-aFNpyKIoXY~Vq-}OWPBe?Rfu^{ps8>Xy%42r@RV#*QV~P83jdlFNgkPN=T|Kt7 zV*M`Rh*30&AWlb$;ae130e@}Tqi3zx2^JQHpM>j$6x`#{mu%tZlwx9Gj@Hc92IuY* zarmT|*d0E~vt6<+r?W^UW0&#U&)8B6+1+;k^2|FWBRP9?C4Rk)HAh&=AS8FS|NQaZ z2j!iZ)nbEyg4ZTp-zHwVlfLC~tXIrv(xrP8PAtR{*c;T24ycA-;auWsya-!kF~CWZ zw_uZ|%urXgUbc@x=L=_g@QJ@m#5beS@6W195Hn7>_}z@Xt{DIEA`A&V82bc^#!q8$ zFh?z_Vn|ozJ;NPd^5uu(9tspo8t%&-U9Ckay-s@DnM*R5rtu|4)~e)`z0P-sy?)kc zs_k&J@0&0!q4~%cKL)2l;N*T&0;mqX5T{Qy60%JtKTQZ-xb%KOcgqwJmb%MOOKk7N zgq})R_6**{8A|6H?fO+2`#QU)p$Ei2&nbj6TpLSIT^D$|`TcSeh+)}VMb}LmvZ{O| ze*1IdCt3+yhdYVxcM)Q_V0bIXLgr6~%JS<<&dxIgfL=Vnx4YHuU@I34JXA|+$_S3~ zy~X#gO_X!cSs^XM{yzDGNM>?v(+sF#<0;AH^YrE8smx<36bUsHbN#y57K8WEu(`qHvQ6cAZPo=J5C(lSmUCZ57Rj6cx!e^rfaI5%w}unz}4 zoX=nt)FVNV%QDJH`o!u9olLD4O5fl)xp+#RloZlaA92o3x4->?rB4`gS$;WO{R;Z3>cG3IgFX2EA?PK^M}@%1%A;?f6}s&CV$cIyEr#q5;yHdNZ9h{| z-=dX+a5elJoDo?Eq&Og!nN6A)5yYpnGEp}?=!C-V)(*~z-+?kY1Q7qs#Rsy%hu_60rdbB+QQNr?S1 z?;xtjUv|*E3}HmuNyB9aFL5H~3Ho0UsmuMZELp1a#CA1g`P{-mT?BchuLEtK}!QZ=3AWakRu~?f9V~3F;TV`5%9Pcs_$gq&CcU}r8gOO zC2&SWPsSG{&o-LIGTBqp6SLQZPvYKp$$7L4WRRZ0BR$Kf0I0SCFkqveCp@f)o8W)! z$%7D1R`&j7W9Q9CGus_)b%+B#J2G;l*FLz#s$hw{BHS~WNLODV#(!u_2Pe&tMsq={ zdm7>_WecWF#D=?eMjLj=-_z`aHMZ=3_-&E8;ibPmM}61i6J3is*=dKf%HC>=xbj4$ zS|Q-hWQ8T5mWde6h@;mS+?k=89?1FU<%qH9B(l&O>k|u_aD|DY*@~(`_pb|B#rJ&g zR0(~(68fpUPz6TdS@4JT5MOPrqDh5_H(eX1$P2SQrkvN8sTxwV>l0)Qq z0pzTuvtEAKRDkKGhhv^jk%|HQ1DdF%5oKq5BS>szk-CIke{%js?~%@$uaN3^Uz6Wf z_iyx{bZ(;9y4X&>LPV=L=d+A}7I4GkK0c1Xts{rrW1Q7apHf-))`BgC^0^F(>At1* za@e7{lq%yAkn*NH8Q1{@{lKhRg*^TfGvv!Sn*ed*x@6>M%aaqySxR|oNadYt1mpUZ z6H(rupHYf&Z z29$5g#|0MX#aR6TZ$@eGxxABRKakDYtD%5BmKp;HbG_ZbT+=81E&=XRk6m_3t9PvD zr5Cqy(v?gHcYvYvXkNH@S#Po~q(_7MOuCAB8G$a9BC##gw^5mW16cML=T=ERL7wsk zzNEayTG?mtB=x*wc@ifBCJ|irFVMOvH)AFRW8WE~U()QT=HBCe@s$dA9O!@`zAAT) zaOZ7l6vyR+Nk_OOF!ZlZmjoImKh)dxFbbR~z(cMhfeX1l7S_`;h|v3gI}n9$sSQ>+3@AFAy9=B_y$)q;Wdl|C-X|VV3w8 z2S#>|5dGA8^9%Bu&fhmVRrTX>Z7{~3V&0UpJNEl0=N32euvDGCJ>#6dUSi&PxFW*s zS`}TB>?}H(T2lxBJ!V#2taV;q%zd6fOr=SGHpoSG*4PDaiG0pdb5`jelVipkEk%FV zThLc@Hc_AL1#D&T4D=w@UezYNJ%0=f3iVRuVL5H?eeZM}4W*bomebEU@e2d`M<~uW zf#Bugwf`VezG|^Qbt6R_=U0}|=k;mIIakz99*>FrsQR{0aQRP6ko?5<7bkDN8evZ& zB@_KqQG?ErKL=1*ZM9_5?Pq%lcS4uLSzN(Mr5=t6xHLS~Ym`UgM@D&VNu8e?_=nSFtF$u@hpPSmI4Vo_t&v?>$~K4y(O~Rb*(MFy_igM7 z*~yYUyR6yQgzWnWMUgDov!!g=lInM+=lOmOk4L`O?{i&qxy&D*_qorRbDwj6?)!ef z#JLd7F6Z2I$S0iYI={rZNk*<{HtIl^mx=h>Cim*04K4+Z4IJtd*-)%6XV2(MCscPiw_a+y*?BKbTS@BZ3AUao^%Zi#PhoY9Vib4N>SE%4>=Jco0v zH_Miey{E;FkdlZSq)e<{`+S3W=*ttvD#hB8w=|2aV*D=yOV}(&p%0LbEWH$&@$X3x~CiF-?ejQ*N+-M zc8zT@3iwkdRT2t(XS`d7`tJQAjRmKAhiw{WOqpuvFp`i@Q@!KMhwKgsA}%@sw8Xo5Y=F zhRJZg)O4uqNWj?V&&vth*H#je6T}}p_<>!Dr#89q@uSjWv~JuW(>FqoJ5^ho0%K?E z9?x_Q;kmcsQ@5=}z@tdljMSt9-Z3xn$k)kEjK|qXS>EfuDmu(Z8|(W?gY6-l z@R_#M8=vxKMAoi&PwnaIYw2COJM@atcgfr=zK1bvjW?9B`-+Voe$Q+H$j!1$Tjn+* z&LY<%)L@;zhnJlB^Og6I&BOR-m?{IW;tyYC%FZ!&Z>kGjHJ6cqM-F z&19n+e1=9AH1VrVeHrIzqlC`w9=*zfmrerF?JMzO&|Mmv;!4DKc(sp+jy^Dx?(8>1 zH&yS_4yL7m&GWX~mdfgH*AB4{CKo;+egw=PrvkTaoBU+P-4u?E|&!c z)DKc;>$$B6u*Zr1SjUh2)FeuWLWHl5TH(UHWkf zLs>7px!c5n;rbe^lO@qlYLzlDVp(z?6rPZel=YB)Uv&n!2{+Mb$-vQl=xKw( zve&>xYx+jW_NJh!FV||r?;hdP*jOXYcLCp>DOtJ?2S^)DkM{{Eb zS$!L$e_o0(^}n3tA1R3-$SNvgBq;DOEo}fNc|tB%%#g4RA3{|euq)p+xd3I8^4E&m zFrD%}nvG^HUAIKe9_{tXB;tl|G<%>yk6R;8L2)KUJw4yHJXUOPM>(-+jxq4R;z8H#>rnJy*)8N+$wA$^F zN+H*3t)eFEgxLw+Nw3};4WV$qj&_D`%ADV2%r zJCPCo%{=z7;`F98(us5JnT(G@sKTZ^;2FVitXyLe-S5(hV&Ium+1pIUB(CZ#h|g)u zSLJJ<@HgrDiA-}V_6B^x1>c9B6%~847JkQ!^KLZ2skm;q*edo;UA)~?SghG8;QbHh z_6M;ouo_1rq9=x$<`Y@EA{C%6-pEV}B(1#sDoe_e1s3^Y>n#1Sw;N|}8D|s|VPd+g z-_$QhCz`vLxxrVMx3ape1xu3*wjx=yKSlM~nFgkNWb4?DDr*!?U)L_VeffF<+!j|b zZ$Wn2$TDv3C3V@BHpSgv3JUif8%hk%OsGZ=OxH@8&4`bbf$`aAMchl^qN>Eyu3JH} z9-S!x8-s4fE=lad%Pkp8hAs~u?|uRnL48O|;*DEU! zuS0{cpk%1E0nc__2%;apFsTm0bKtd&A0~S3Cj^?72-*Owk3V!ZG*PswDfS~}2<8le z5+W^`Y(&R)yVF*tU_s!XMcJS`;(Tr`J0%>p=Z&InR%D3@KEzzI+-2)HK zuoNZ&o=wUC&+*?ofPb0a(E6(<2Amd6%uSu_^-<1?hsxs~0K5^f(LsGqgEF^+0_H=uNk9S0bb!|O8d?m5gQjUKevPaO+*VfSn^2892K~%crWM8+6 z25@V?Y@J<9w%@NXh-2!}SK_(X)O4AM1-WTg>sj1{lj5@=q&dxE^9xng1_z9w9DK>| z6Iybcd0e zyi;Ew!KBRIfGPGytQ6}z}MeXCfLY0?9%RiyagSp_D1?N&c{ zyo>VbJ4Gy`@Fv+5cKgUgs~na$>BV{*em7PU3%lloy_aEovR+J7TfQKh8BJXyL6|P8un-Jnq(ghd!_HEOh$zlv2$~y3krgeH;9zC}V3f`uDtW(%mT#944DQa~^8ZI+zAUu4U(j0YcDfKR$bK#gvn_{JZ>|gZ5+)u?T$w7Q%F^;!Wk?G z(le7r!ufT*cxS}PR6hIVtXa)i`d$-_1KkyBU>qmgz-=T};uxx&sKgv48akIWQ89F{ z0XiY?WM^~;|T8zBOr zs#zuOONzH?svv*jokd5SK8wG>+yMC)LYL|vLqm^PMHcT=`}V$=nIRHe2?h)8WQa6O zPAU}d`1y(>kZiP~Gr=mtJLMu`i<2CspL|q2DqAgAD^7*$xzM`PU4^ga`ilE134XBQ z99P(LhHU@7qvl9Yzg$M`+dlS=x^(m-_3t|h>S}E0bcFMn=C|KamQ)=w2^e)35p`zY zRV8X?d;s^>Cof2SPR&nP3E+-LCkS0J$H!eh8~k0qo$}00b=7!H_I2O+Ro@3O$nPdm ztmbOO^B+IHzQ5w>@@@J4cKw5&^_w6s!s=H%&byAbUtczPQ7}wfTqxxtQNfn*u73Qw zGuWsrky_ajPx-5`R<)6xHf>C(oqGf_Fw|-U*GfS?xLML$kv;h_pZ@Kk$y0X(S+K80 z6^|z)*`5VUkawg}=z`S;VhZhxyDfrE0$(PMurAxl~<>lfZa>JZ288ULK7D` zl9|#L^JL}Y$j*j`0-K6kH#?bRmg#5L3iB4Z)%iF@SqT+Lp|{i`m%R-|ZE94Np7Pa5 zCqC^V3}B(FR340pmF*qaa}M}+h6}mqE~7Sh!9bDv9YRT|>vBNAqv09zXHMlcuhKD| zcjjA(b*XCIwJ33?CB!+;{)vX@9xns_b-VO{i0y?}{!sdXj1GM8+$#v>W7nw;+O_9B z_{4L;C6ol?(?W0<6taGEn1^uG=?Q3i29sE`RfYCaV$3DKc_;?HsL?D_fSYg}SuO5U zOB_f4^vZ_x%o`5|C@9C5+o=mFy@au{s)sKw!UgC&L35aH(sgDxRE2De%(%OT=VUdN ziVLEmdOvJ&5*tCMKRyXctCwQu_RH%;m*$YK&m;jtbdH#Ak~13T1^f89tn`A%QEHWs~jnY~E}p_Z$XC z=?YXLCkzVSK+Id`xZYTegb@W8_baLt-Fq`Tv|=)JPbFsKRm)4UW;yT+J`<)%#ue9DPOkje)YF2fsCilK9MIIK>p*`fkoD5nGfmLwt)!KOT+> zOFq*VZktDDyM3P5UOg`~XL#cbzC}eL%qMB=Q5$d89MKuN#$6|4gx_Jt0Gfn8w&q}%lq4QU%6#jT*MRT% zrLz~C8FYKHawn-EQWN1B75O&quS+Z81(zN)G>~vN8VwC+e+y(`>HcxC{MrJ;H1Z4k zZWuv$w_F0-Ub%MVcpIc){4PGL^I7M{>;hS?;eH!;gmcOE66z3;Z1Phqo(t zVP(Hg6q#0gIKgsg7L7WE!{Y#1nI(45tx2{$34dDd#!Z0NIyrm)HOn5W#7;f4pQci# zDW!FI(g4e668kI9{2+mLwB+=#9bfqgX%!B34V-$wwSN(_cm*^{y0jQtv*4}eO^sOV z*9xoNvX)c9isB}Tgx&ZRjp3kwhTVK?r9;n!x>^XYT z@Q^7zp{rkIs{2mUSE^2!Gf6$6;j~&4=-0cSJJDizZp6LTe8b45;{AKM%v99}{{FfC zz709%u0mC=1KXTo(=TqmZQ;c?$M3z(!xah>aywrj40sc2y3rKFw4jCq+Y+u=CH@_V zxz|qeTwa>+<|H%8Dz5u>ZI5MmjTFwXS-Fv!TDd*`>3{krWoNVx$<133`(ftS?ZPyY z&4@ah^3^i`vL$BZa>O|Nt?ucewzsF)0zX3qmM^|waXr=T0pfIb0*$AwU=?Ipl|1Y; z*Pk6{C-p4MY;j@IJ|DW>QHZQJcp;Z~?8(Q+Kk3^0qJ}SCk^*n4W zu9ZFwLHUx-$6xvaQ)SUQcYd6fF8&x)V`1bIuX@>{mE$b|Yd(qomn3;bPwnDUc0F=; zh*6_((%bqAYQWQ~odER?h>1mkL4kpb3s7`0m@rDKGU*oyF)$j~Ffd4fXV$?`f~rHf zB%Y)@5SXZvfwm10RY5X?TEo)PK_`L6qgBp=#>fO49$D zDq8Ozj0q6213tV5Qq=;fZ0$|KroY{Dz=l@lU^J)?Ko@ti20TRplXzphBi>XGx4bou zEWrkNjz0t5j!_ke{g5I#PUlEU$Km8g8TE|XK=MkU@PT4T><2OVamoK;wJ}3X0L$vX zgd7gNa359*nc)R-0!`2X@FOTB`+oETOPc=ubp5R)VQgY+5BTZZJ2?9QwnO=dnulIUF3gFn;BODC2)65)HeVd%t86sL7Rv^Y+nbn+&l z6BAJY(ETvwI)Ts$aiE8rht4KD*qNyE{8{x6R|%akbTBzw;2+6Echkt+W+`u^XX z_z&x%n@00IL3?^hro$gg*4VI_WAaTyVM5Foj~O|-84 z$;06hMwt*rV;^8iB z1~&0XWpYJmG?Ts^K9PC62H*`G}xom%S%yq|xvG~FIfP=9*f zZoDRJBm*Y0aId=qJ?7dyb)6)JGWGwe)MHeNSzhi)Ko6J<-m@v=a%NsP537lHe0R* z`If4$aaBA#S=w!2z&m>{lpTy^Lm^mg*3?M&7HFv}7K6x*cukLIGX;bQG|QWdn{%_6 zHnwBKr84#B7Z+AnBXa16a?or^R?+>$4`}{*a_>IhbjvyTtWkHw)|ay)ahWUd-qq$~ zMbh6roVsj;_qnC-R{G+Cy6bApVOinSU-;(DxUEl!i2)1EeQ9`hrfqj(nKI7?Z>Xur zoJz-a`PxkYit1HEbv|jy%~DO^13J-ut986EEG=66S}D3!L}Efp;Bez~7tNq{QsUMm zh9~(HYg1pA*=37C0}n4g&bFbQ+?-h-W}onYeE{q;cIy%eZK9wZjSwGvT+&Cgv z?~{9p(;bY_1+k|wkt_|N!@J~aoY@|U_RGoWX<;p{Nu*D*&_phw`8jYkMNpRTWx1H* z>J-Mi_!`M468#5Aix$$u1M@rJEIOc?k^QBc?T(#=n&*5eS#u*Y)?L8Ha$9wRWdH^3D4|Ps)Y?m0q~SiKiSfEkJ!=^`lJ(%W3o|CZ zSrZL-Xxc{OrmsQD&s~zPfNJOpSZUl%V8tdG%ei}lQkM+z@-4etFPR>GOH9+Y_F<3=~SXln9Kb-o~f>2a6Xz@AS3cn^;c_>lUwlK(n>z?A>NbC z`Ud8^aQy>wy=$)w;JZzA)_*Y$Z5hU=KAG&htLw1Uh00yE!|Nu{EZkch zY9O6x7Y??>!7pUNME*d!=R#s)ghr|R#41l!c?~=3CS8&zr6*aA7n9*)*PWBV2w+&I zpW1-9fr3j{VTcls1>ua}F*bbju_Xq%^v;-W~paSqlf zolj*dt`BBjHI)H9{zrkBo=B%>8}4jeBO~kWqO!~Thi!I1H(in=n^fS%nuL=X2+s!p}HfTU#NBGiwEBF^^tKU zbhhv+0dE-sbK$>J#t-J!B$TMgN@Wh5wTtK2BG}4BGfsZOoRUS#G8Cxv|6EI*n&Xxq zt{&OxCC+BNqz$9b0WM7_PyBJEVObHFh%%`~!@MNZlo*oXDCwDcFwT~Rls!aApL<)^ zbBftGKKBRhB!{?fX@l2_y~%ygNFfF(XJzHh#?`WlSL{1lKT*gJM zs>bd^H9NCxqxn(IOky5k-wALFowQr(gw%|`0991u#9jXQh?4l|l>pd6a&rx|v=fPJ z1mutj{YzpJ_gsClbWFk(G}bSlFi-6@mwoQh-XeD*j@~huW4(8ub%^I|azA)h2t#yG z7e_V_<4jlM3D(I+qX}yEtqj)cpzN*oCdYHa!nm%0t^wHm)EmFP*|FMw!tb@&`G-u~ zK)=Sf6z+BiTAI}}i{*_Ac$ffr*Wrv$F7_0gJkjx;@)XjYSh`RjAgrCck`x!zP>Ifu z&%he4P|S)H*(9oB4uvH67^0}I-_ye_!w)u3v2+EY>eD3#8QR24<;7?*hj8k~rS)~7 zSXs5ww)T(0eHSp$hEIBnW|Iun<_i`}VE0Nc$|-R}wlSIs5pV{g_Dar(Zz<4X3`W?K z6&CAIl4U(Qk-tTcK{|zYF6QG5ArrEB!;5s?tW7 zrE3hcFY&k)+)e{+YOJ0X2uDE_hd2{|m_dC}kgEKqiE9Q^A-+>2UonB+L@v3$9?AYw zVQv?X*pK;X4Ovc6Ev5Gbg{{Eu*7{N3#0@9oMI~}KnObQE#Y{&3mM4`w%wN+xrKYgD zB-ay0Q}m{QI;iY`s1Z^NqIkjrTlf`B)B#MajZ#9u41oRBC1oM1vq0i|F59> z#StM@bHt|#`2)cpl_rWB($DNJ3Lap}QM-+A$3pe}NyP(@+i1>o^fe-oxX#Bt`mcQc zb?pD4W%#ep|3%CHAYnr*^M6Czg>~L4?l16H1OozM{P*en298b+`i4$|w$|4AHbzqB zHpYUsHZET$Z0ztC;U+0*+amF!@PI%^oUIZy{`L{%O^i{Xk}X0&nl)n~tVEpcAJSJ} zverw15zP1P-O8h9nd!&hj$zuwjg?DoxYIw{jWM zW5_pj+wFy8Tsa9g<7Qa21WaV&;ejoYflRKcz?#fSH_)@*QVlN2l4(QNk| z4aPnv&mrS&0|6NHq05XQw$J^RR9T{3SOcMKCXIR1iSf+xJ0E_Wv?jEc*I#ZPzyJN2 zUG0UOXHl+PikM*&g$U@g+KbG-RY>uaIl&DEtw_Q=FYq?etc!;hEC_}UX{eyh%dw2V zTTSlap&5>PY{6I#(6`j-9`D&I#|YPP8a;(sOzgeKDWsLa!i-$frD>zr-oid!Hf&yS z!i^cr&7tN}OOGmX2)`8k?Tn!!4=tz~3hCTq_9CdiV!NIblUDxHh(FJ$zs)B2(t5@u z-`^RA1ShrLCkg0)OhfoM;4Z{&oZmAec$qV@ zGQ(7(!CBk<5;Ar%DLJ0p0!ResC#U<+3i<|vib1?{5gCebG7$F7URKZXuX-2WgF>YJ^i zMhHDBsh9PDU8dlZ$yJKtc6JA#y!y$57%sE>4Nt+wF1lfNIWyA`=hF=9Gj%sRwi@vd z%2eVV3y&dvAgyuJ=eNJR+*080dbO_t@BFJO<@&#yqTK&+xc|FRR;p;KVk@J3$S{p` zGaMj6isho#%m)?pOG^G0mzOAw0z?!AEMsv=0T>WWcE>??WS=fII$t$(^PDPMU(P>o z_*0s^W#|x)%tx8jIgZY~A2yG;US0m2ZOQt6yJqW@XNY_>_R7(Nxb8Ged6BdYW6{prd!|zuX$@Q2o6Ona8zzYC1u!+2!Y$Jc9a;wy+pXt}o6~Bu1oF1c zp7Y|SBTNi@=I(K%A60PMjM#sfH$y*c{xUgeSpi#HB`?|`!Tb&-qJ3;vxS!TIzuTZs-&%#bAkAyw9m4PJgvey zM5?up*b}eDEY+#@tKec)-c(#QF0P?MRlD1+7%Yk*jW;)`f;0a-ZJ6CQA?E%>i2Dt7T9?s|9ZF|KP4;CNWvaVKZ+Qeut;Jith_y{v*Ny6Co6!8MZx;Wgo z=qAi%&S;8J{iyD&>3CLCQdTX*$+Rx1AwA*D_J^0>suTgBMBb=*hefV+Ars#mmr+YsI3#!F@Xc1t4F-gB@6aoyT+5O(qMz*zG<9Qq*f0w^V!03rpr*-WLH}; zfM{xSPJeu6D(%8HU%0GEa%waFHE$G?FH^kMS-&I3)ycx|iv{T6Wx}9$$D&6{%1N_8 z_CLw)_9+O4&u94##vI9b-HHm_95m)fa??q07`DniVjAy`t7;)4NpeyAY(aAk(+T_O z1om+b5K2g_B&b2DCTK<>SE$Ode1DopAi)xaJjU>**AJK3hZrnhEQ9E`2=|HHe<^tv z63e(bn#fMWuz>4erc47}!J>U58%<&N<6AOAewyzNTqi7hJc|X{782&cM zHZYclNbBwU6673=!ClmxMfkC$(CykGR@10F!zN1Se83LR&a~$Ht&>~43OX22mt7tcZUpa;9@q}KDX3O&Ugp6< zLZLfIMO5;pTee1vNyVC$FGxzK2f>0Z-6hM82zKg44nWo|n}$Zk6&;5ry3`(JFEX$q zK&KivAe${e^5ZGc3a9hOt|!UOE&OocpVryE$Y4sPcs4rJ>>Kbi2_subQ9($2VN(3o zb~tEzMsHaBmBtaHAyES+d3A(qURgiskSSwUc9CfJ@99&MKp2sooSYZu+-0t0+L*!I zYagjOlPgx|lep9tiU%ts&McF6b0VE57%E0Ho%2oi?=Ks+5%aj#au^OBwNwhec zta6QAeQI^V!dF1C)>RHAmB`HnxyqWx?td@4sd15zPd*Fc9hpDXP23kbBenBxGeD$k z;%0VBQEJ-C)&dTAw_yW@k0u?IUk*NrkJ)(XEeI z9Y>6Vel>#s_v@=@0<{4A{pl=9cQ&Iah0iD0H`q)7NeCIRz8zx;! z^OO;1+IqoQNak&pV`qKW+K0^Hqp!~gSohcyS)?^P`JNZXw@gc6{A3OLZ?@1Uc^I2v z+X!^R*HCm3{7JPq{8*Tn>5;B|X7n4QQ0Bs79uTU%nbqOJh`nX(BVj!#f;#J+WZxx4 z_yM&1Y`2XzhfqkIMO7tB3raJKQS+H5F%o83bM+hxbQ zeeJm=Dvix$2j|b4?mDacb67v-1^lTp${z=jc1=j~QD>7c*@+1?py>%Kj%Ejp7Y-!? z8iYRUlGVrQPandAaxFfks53@2EC#0)%mrnmGRn&>=$H$S8q|kE_iWko4`^vCS2aWg z#!`RHUGyOt*k?bBYu3*j3u0gB#v(3tsije zgIuNNWNtrOkx@Pzs;A9un+2LX!zw+p3_NX^Sh09HZAf>m8l@O*rXy_82aWT$Q>iyy zqO7Of)D=wcSn!0+467&!Hl))eff=$aneB?R!YykdKW@k^_uR!+Q1tR)+IJb`-6=jj zymzA>Sv4>Z&g&WWu#|~GcP7qP&m*w-S$)7Xr;(duqCTe7p8H3k5>Y-n8438+%^9~K z3r^LIT_K{i7DgEJjIocw_6d0!<;wKT`X;&vv+&msmhAAnIe!OTdybPctzcEzBy88_ zWO{6i4YT%e4^WQZB)KHCvA(0tS zHu_Bg+6Ko%a9~$EjRB90`P(2~6uI@SFibxct{H#o&y40MdiXblu@VFXbhz>Nko;7R z70Ntmm-FePqhb%9gL+7U8@(ch|JfH5Fm)5${8|`Lef>LttM_iww6LW2X61ldBmG0z zax3y)njFe>j*T{i0s8D4=L>X^j0)({R5lMGVS#7(2C9@AxL&C-lZQx~czI7Iv+{%1 z2hEG>RzX4S8x3v#9sgGAnPzptM)g&LB}@%E>fy0vGSa(&q0ch|=ncKjNrK z`jA~jObJhrJ^ri|-)J^HUyeZXz~XkBp$VhcTEcTdc#a2EUOGVX?@mYx#Vy*!qO$Jv zQ4rgOJ~M*o-_Wptam=~krnmG*p^j!JAqoQ%+YsDFW7Cc9M%YPiBOrVcD^RY>m9Pd< zu}#9M?K{+;UIO!D9qOpq9yxUquQRmQNMo0pT`@$pVt=rMvyX)ph(-CCJLvUJy71DI zBk7oc7)-%ngdj~s@76Yse3L^gV0 z2==qfp&Q~L(+%RHP0n}+xH#k(hPRx(!AdBM$JCfJ5*C=K3ts>P?@@SZ_+{U2qFZb>4kZ{Go37{# zSQc+-dq*a-Vy4?taS&{Ht|MLRiS)Sn14JOONyXqPNnpq&2y~)6wEG0oNy>qvod$FF z`9o&?&6uZjhZ4_*5qWVrEfu(>_n2Xi2{@Gz9MZ8!YmjYvIMasE9yVQL10NBrTCczq zcTY1q^PF2l!Eraguf{+PtHV3=2A?Cu&NN&a8V(y;q(^_mFc6)%Yfn&X&~Pq zU1?qCj^LF(EQB1F`8NxNjyV%fde}dEa(Hx=r7$~ts2dzDwyi6ByBAIx$NllB4%K=O z$AHz1<2bTUb>(MCVPpK(E9wlLElo(aSd(Os)^Raum`d(g9Vd_+Bf&V;l=@mM=cC>) z)9b0enb)u_7V!!E_bl>u5nf&Rl|2r=2F3rHMdb7y9E}}F82^$Rf+P8%dKnOeKh1vs zhH^P*4Ydr^$)$h@4KVzxrHyy#cKmWEa9P5DJ|- zG;!Qi35Tp7XNj60=$!S6U#!(${6hyh7d4q=pF{`0t|N^|L^d8pD{O9@tF~W;#Je*P z&ah%W!KOIN;SyAEhAeTafJ4uEL`(RtnovM+cb(O#>xQnk?dzAjG^~4$dFn^<@-Na3 z395;wBnS{t*H;Jef2eE!2}u5Ns{AHj>WYZDgQJt8v%x?9{MXqJsGP|l%OiZqQ1aB! z%E=*Ig`(!tHh>}4_z5IMpg{49UvD*Pp9!pxt_gdAW%sIf3k6CTycOT1McPl=_#0?8 zVjz8Hj*Vy9c5-krd-{BQ{6Xy|P$6LJvMuX$* zA+@I_66_ET5l2&gk9n4$1M3LN8(yEViRx&mtd#LD}AqEs?RW=xKC(OCWH;~>(X6h!uDxXIPH06xh z*`F4cVlbDP`A)-fzf>MuScYsmq&1LUMGaQ3bRm6i7OsJ|%uhTDT zlvZA1M}nz*SalJWNT|`dBm1$xlaA>CCiQ zK`xD-RuEn>-`Z?M{1%@wewf#8?F|(@1e0+T4>nmlSRrNK5f)BJ2H*$q(H>zGD0>eL zQ!tl_Wk)k*e6v^m*{~A;@6+JGeWU-q9>?+L_#UNT%G?4&BnOgvm9@o7l?ov~XL+et zbGT)|G7)KAeqb=wHSPk+J1bdg7N3$vp(ekjI1D9V$G5Cj!=R2w=3*4!z*J-r-cyeb zd(i2KmX!|Lhey!snRw z?#$Gu%S^SQEKt&kep)up#j&9}e+3=JJBS(s>MH+|=R(`8xK{mmndWo_r`-w1#SeRD&YtAJ#GiVI*TkQZ}&aq<+bU2+coU3!jCI6E+Ad_xFW*ghnZ$q zAoF*i&3n1j#?B8x;kjSJD${1jdRB;)R*)Ao!9bd|C7{;iqDo|T&>KSh6*hCD!rwv= zyK#F@2+cv3=|S1Kef(E6Niv8kyLVLX&e=U;{0x{$tDfShqkjUME>f8d(5nzSkY6@! z^-0>DM)wa&%m#UF1F?zR`8Y3X#tA!*7Q$P3lZJ%*KNlrk_uaPkxw~ zxZ1qlE;Zo;nb@!SMazSjM>;34ROOoygo%SF);LL>rRonWwR>bmSd1XD^~sGSu$Gg# zFZ`|yKU0%!v07dz^v(tY%;So(e`o{ZYTX`hm;@b0%8|H>VW`*cr8R%3n|ehw2`(9B+V72`>SY}9^8oh$En80mZK9T4abVG*to;E z1_S6bgDOW?!Oy1LwYy=w3q~KKdbNtyH#d24PFjX)KYMY93{3-mPP-H>@M-_>N~DDu zENh~reh?JBAK=TFN-SfDfT^=+{w4ea2KNWXq2Y<;?(gf(FgVp8Zp-oEjKzB%2Iqj;48GmY3h=bcdYJ}~&4tS`Q1sb=^emaW$IC$|R+r-8V- zf0$gGE(CS_n4s>oicVk)MfvVg#I>iDvf~Ov8bk}sSxluG!6#^Z_zhB&U^`eIi1@j( z^CK$z^stBHtaDDHxn+R;3u+>Lil^}fj?7eaGB z&5nl^STqcaBxI@v>%zG|j))G(rVa4aY=B@^2{TFkW~YP!8!9TG#(-nOf^^X-%m9{Z zCC?iC`G-^RcBSCuk=Z`(FaUUe?hf3{0C>>$?Vs z`2Uud9M+T&KB6o4o9kvdi^Q=Bw!asPdxbe#W-Oaa#_NP(qpyF@bVxv5D5))srkU#m zj_KA+#7sqDn*Ipf!F5Byco4HOSd!Ui$l94|IbW%Ny(s1>f4|Mv^#NfB31N~kya9!k zWCGL-$0ZQztBate^fd>R!hXY_N9ZjYp3V~4_V z#eB)Kjr8yW=+oG)BuNdZG?jaZlw+l_ma8aET(s+-x+=F-t#Qoiuu1i`^x8Sj>b^U} zs^z<()YMFP7CmjUC@M=&lA5W7t&cxTlzJAts*%PBDAPuqcV5o7HEnqjif_7xGt)F% zGx2b4w{@!tE)$p=l3&?Bf#`+!-RLOleeRk3 z7#pF|w@6_sBmn1nECqdunmG^}pr5(ZJQVvAt$6p3H(16~;vO>?sTE`Y+mq5YP&PBo zvq!7#W$Gewy`;%6o^!Dtjz~x)T}Bdk*BS#=EY=ODD&B=V6TD2z^hj1m5^d6s)D*wk zu$z~D7QuZ2b?5`p)E8e2_L38v3WE{V`bVk;6fl#o2`) z99JsWhh?$oVRn@$S#)uK&8DL8>An0&S<%V8hnGD7Z^;Y(%6;^9!7kDQ5bjR_V+~wp zfx4m3z6CWmmZ<8gDGUyg3>t8wgJ5NkkiEm^(sedCicP^&3D%}6LtIUq>mXCAt{9eF zNXL$kGcoUTf_Lhm`t;hD-SE)m=iBnxRU(NyL}f6~1uH)`K!hmYZjLI%H}AmEF5RZt z06$wn63GHnApHXZZJ}s^s)j9(BM6e*7IBK6Bq(!)d~zR#rbxK9NVIlgquoMq z=eGZ9NR!SEqP6=9UQg#@!rtbbSBUM#ynF);zKX+|!Zm}*{H z+j=d?aZ2!?@EL7C~%B?6ouCKLnO$uWn;Y6Xz zX8dSwj732u(o*U3F$F=7xwxm>E-B+SVZH;O-4XPuPkLSt_?S0)lb7EEg)Mglk0#eS z9@jl(OnH4juMxY+*r03VDfPx_IM!Lmc(5hOI;`?d37f>jPP$?9jQQIQU@i4vuG6MagEoJrQ=RD7xt@8E;c zeGV*+Pt+t$@pt!|McETOE$9k=_C!70uhwRS9X#b%ZK z%q(TIUXSS^F0`4Cx?Rk07C6wI4!UVPeI~-fxY6`YH$kABdOuiRtl73MqG|~AzZ@iL&^s?24iS;RK_pdlWkhcF z@Wv-Om(Aealfg)D^adlXh9Nvf~Uf@y;g3Y)i(YP zEXDnb1V}1pJT5ZWyw=1i+0fni9yINurD=EqH^ciOwLUGi)C%Da)tyt=zq2P7pV5-G zR7!oq28-Fgn5pW|nlu^b!S1Z#r7!Wtr{5J5PQ>pd+2P7RSD?>(U7-|Y z7ZQ5lhYIl_IF<9?T9^IPK<(Hp;l5bl5tF9>X-zG14_7PfsA>6<$~A338iYRT{a@r_ zuXBaT=`T5x3=s&3=RYx6NgG>No4?5KFBVjE(swfcivcIpPQFx5l+O;fiGsOrl5teR z_Cm+;PW}O0Dwe_(4Z@XZ)O0W-v2X><&L*<~*q3dg;bQW3g7)a#3KiQP>+qj|qo*Hk z?57>f2?f@`=Fj^nkDKeRkN2d$Z@2eNKpHo}ksj-$`QKb6n?*$^*%Fb3_Kbf1(*W9K>{L$mud2WHJ=j0^=g30Xhg8$#g^?36`p1fm;;1@0Lrx+8t`?vN0ZorM zSW?rhjCE8$C|@p^sXdx z|NOHHg+fL;HIlqyLp~SSdIF`TnSHehNCU9t89yr@)FY<~hu+X`tjg(aSVae$wDG*C zq$nY(Y494R)hD!i1|IIyP*&PD_c2FPgeY)&mX1qujB1VHPG9`yFQpLFVQ0>EKS@Bp zAfP5`C(sWGLI?AC{XEjLKR4FVNw(4+9b?kba95ukgR1H?w<8F7)G+6&(zUhIE5Ef% z=fFkL3QKA~M@h{nzjRq!Y_t!%U66#L8!(2-GgFxkD1=JRRqk=n%G(yHKn%^&$dW>; zSjAcjETMz1%205se$iH_)ZCpfg_LwvnsZQAUCS#^FExp8O4CrJb6>JquNV@qPq~3A zZ<6dOU#6|8+fcgiA#~MDmcpIEaUO02L5#T$HV0$EMD94HT_eXLZ2Zi&(! z&5E>%&|FZ`)CN10tM%tLSPD*~r#--K(H-CZqIOb99_;m|D5wdgJ<1iOJz@h2Zkq?} z%8_KXb&hf=2Wza(Wgc;3v3TN*;HTU*q2?#z&tLn_U0Nt!y>Oo>+2T)He6%XuP;fgn z-G!#h$Y2`9>Jtf}hbVrm6D70|ERzLAU>3zoWhJmjWfgM^))T+2u$~5>HF9jQDkrXR z=IzX36)V75PrFjkQ%TO+iqKGCQ-DDXbaE;C#}!-CoWQx&v*vHfyI>$HNRbpvm<`O( zlx9NBWD6_e&J%Ous4yp~s6)Ghni!I6)0W;9(9$y1wWu`$gs<$9Mcf$L*piP zPR0Av*2%ul`W;?-1_-5Zy0~}?`e@Y5A&0H!^ApyVTT}BiOm4GeFo$_oPlDEyeGBbh z1h3q&Dx~GmUS|3@4V36&$2uO8!Yp&^pD7J5&TN{?xphf*-js1fP?B|`>p_K>lh{ij zP(?H%e}AIP?_i^f&Li=FDSQ`2_NWxL+BB=nQr=$ zHojMlXNGauvvwPU>ZLq!`bX-5F4jBJ&So{kE5+ms9UEYD{66!|k~3vsP+mE}x!>%P za98bAU0!h0&ka4EoiDvBM#CP#dRNdXJcb*(%=<(g+M@<)DZ!@v1V>;54En?igcHR2 zhubQMq}VSOK)onqHfczM7YA@s=9*ow;k;8)&?J3@0JiGcP! zP#00KZ1t)GyZeRJ=f0^gc+58lc4Qh*S7RqPIC6GugG1gXe$LIQMRCo8cHf^qXgAa2 z`}t>u2Cq1CbSEpLr~E=c7~=Qkc9-vLE%(v9N*&HF`(d~(0`iukl5aQ9u4rUvc8%m) zr2GwZN4!s;{SB87lJB;veebPmqE}tSpT>+`t?<457Q9iV$th%i__Z1kOMAswFldD6 ztbOvO337S5o#ZZgN2G99_AVqPv!?Gmt3pzgD+Hp3QPQ`9qJ(g=kjvD+fUSS3upJn! zqoG7acIKEFRX~S}3|{EWT$kdz#zrDlJU(rPkxjws_iyLKU8+v|*oS_W*-guAb&Pj1 z35Z`3z<&Jb@2Mwz=KXucNYdY#SNO$tcVFr9KdKm|%^e-TXzs6M`PBper%ajkrIyUe zp$vVxVs9*>Vp4_1NC~Zg)WOCPmOxI1V34QlG4!aSFOH{QqSVq1^1)- z0P!Z?tT&E-ll(pwf0?=F=yOzik=@nh1Clxr9}Vij89z)ePDSCYAqw?lVI?v?+&*zH z)p$CScFI8rrwId~`}9YWPFu0cW1Sf@vRELs&cbntRU6QfPK-SO*mqu|u~}8AJ!Q$z znzu}50O=YbjwKCuSVBs6&CZR#0FTu)3{}qJJYX(>QPr4$RqWiwX3NT~;>cLn*_&1H zaKpIW)JVJ>b{uo2oq>oQt3y=zJjb%fU@wLqM{SyaC6x2snMx-}ivfU<1- znu1Lh;i$3Tf$Kh5Uk))G!D1UhE8pvx&nO~w^fG)BC&L!_hQk%^p`Kp@F{cz>80W&T ziOK=Sq3fdRu*V0=S53rcIfWFazI}Twj63CG(jOB;$*b`*#B9uEnBM`hDk*EwSRdwP8?5T?xGUKs=5N83XsR*)a4|ijz|c{4tIU+4j^A5C<#5 z*$c_d=5ml~%pGxw#?*q9N7aRwPux5EyqHVkdJO=5J>84!X6P>DS8PTTz>7C#FO?k#edkntG+fJk8ZMn?pmJSO@`x-QHq;7^h6GEXLXo1TCNhH z8ZDH{*NLAjo3WM`xeb=X{((uv3H(8&r8fJJg_uSs_%hOH%JDD?hu*2NvWGYD+j)&` zz#_1%O1wF^o5ryt?O0n;`lHbzp0wQ?rcbW(F1+h7_EZZ9{>rePvLAPVZ_R|n@;b$;UchU=0j<6k8G9QuQf@76oiE*4 zXOLQ&n3$NR#p4<5NJMVC*S);5x2)eRbaAM%VxWu9ohlT;pGEk7;002enCbQ>2r-us z3#bpXP9g|mE`65VrN`+3mC)M(eMj~~eOf)do<@l+fMiTR)XO}422*1SL{wyY(%oMpBgJagtiDf zz>O6(m;};>Hi=t8o{DVC@YigqS(Qh+ix3Rwa9aliH}a}IlOCW1@?%h_bRbq-W{KHF z%Vo?-j@{Xi@=~Lz5uZP27==UGE15|g^0gzD|3x)SCEXrx`*MP^FDLl%pOi~~Il;dc z^hrwp9sYeT7iZ)-ajKy@{a`kr0-5*_!XfBpXwEcFGJ;%kV$0Nx;apKrur zJN2J~CAv{Zjj%FolyurtW8RaFmpn&zKJWL>(0;;+q(%(Hx!GMW4AcfP0YJ*Vz!F4g z!ZhMyj$BdXL@MlF%KeInmPCt~9&A!;cRw)W!Hi@0DY(GD_f?jeV{=s=cJ6e}JktJw zQORnxxj3mBxfrH=x{`_^Z1ddDh}L#V7i}$njUFRVwOX?qOTKjfPMBO4y(WiU<)epb zvB9L=%jW#*SL|Nd_G?E*_h1^M-$PG6Pc_&QqF0O-FIOpa4)PAEPsyvB)GKasmBoEt z?_Q2~QCYGH+hW31x-B=@5_AN870vY#KB~3a*&{I=f);3Kv7q4Q7s)0)gVYx2#Iz9g(F2;=+Iy4 z6KI^8GJ6D@%tpS^8boU}zpi=+(5GfIR)35PzrbuXeL1Y1N%JK7PG|^2k3qIqHfX;G zQ}~JZ-UWx|60P5?d1e;AHx!_;#PG%d=^X(AR%i`l0jSpYOpXoKFW~7ip7|xvN;2^? zsYC9fanpO7rO=V7+KXqVc;Q5z%Bj})xHVrgoR04sA2 zl~DAwv=!(()DvH*=lyhIlU^hBkA0$e*7&fJpB0|oB7)rqGK#5##2T`@_I^|O2x4GO z;xh6ROcV<9>?e0)MI(y++$-ksV;G;Xe`lh76T#Htuia+(UrIXrf9?

L(tZ$0BqX1>24?V$S+&kLZ`AodQ4_)P#Q3*4xg8}lMV-FLwC*cN$< zt65Rf%7z41u^i=P*qO8>JqXPrinQFapR7qHAtp~&RZ85$>ob|Js;GS^y;S{XnGiBc zGa4IGvDl?x%gY`vNhv8wgZnP#UYI-w*^4YCZnxkF85@ldepk$&$#3EAhrJY0U)lR{F6sM3SONV^+$;Zx8BD&Eku3K zKNLZyBni3)pGzU0;n(X@1fX8wYGKYMpLmCu{N5-}epPDxClPFK#A@02WM3!myN%bkF z|GJ4GZ}3sL{3{qXemy+#Uk{4>Kf8v11;f8I&c76+B&AQ8udd<8gU7+BeWC`akUU~U zgXoxie>MS@rBoyY8O8Tc&8id!w+_ooxcr!1?#rc$-|SBBtH6S?)1e#P#S?jFZ8u-Bs&k`yLqW|{j+%c#A4AQ>+tj$Y z^CZajspu$F%73E68Lw5q7IVREED9r1Ijsg#@DzH>wKseye>hjsk^{n0g?3+gs@7`i zHx+-!sjLx^fS;fY!ERBU+Q zVJ!e0hJH%P)z!y%1^ZyG0>PN@5W~SV%f>}c?$H8r;Sy-ui>aruVTY=bHe}$e zi&Q4&XK!qT7-XjCrDaufT@>ieQ&4G(SShUob0Q>Gznep9fR783jGuUynAqc6$pYX; z7*O@@JW>O6lKIk0G00xsm|=*UVTQBB`u1f=6wGAj%nHK_;Aqmfa!eAykDmi-@u%6~ z;*c!pS1@V8r@IX9j&rW&d*}wpNs96O2Ute>%yt{yv>k!6zfT6pru{F1M3P z2WN1JDYqoTB#(`kE{H676QOoX`cnqHl1Yaru)>8Ky~VU{)r#{&s86Vz5X)v15ULHA zAZDb{99+s~qI6;-dQ5DBjHJP@GYTwn;Dv&9kE<0R!d z8tf1oq$kO`_sV(NHOSbMwr=To4r^X$`sBW4$gWUov|WY?xccQJN}1DOL|GEaD_!@& z15p?Pj+>7d`@LvNIu9*^hPN)pwcv|akvYYq)ks%`G>!+!pW{-iXPZsRp8 z35LR;DhseQKWYSD`%gO&k$Dj6_6q#vjWA}rZcWtQr=Xn*)kJ9kacA=esi*I<)1>w^ zO_+E>QvjP)qiSZg9M|GNeLtO2D7xT6vsj`88sd!94j^AqxFLi}@w9!Y*?nwWARE0P znuI_7A-saQ+%?MFA$gttMV-NAR^#tjl_e{R$N8t2NbOlX373>e7Ox=l=;y#;M7asp zRCz*CLnrm$esvSb5{T<$6CjY zmZ(i{Rs_<#pWW>(HPaaYj`%YqBra=Ey3R21O7vUbzOkJJO?V`4-D*u4$Me0Bx$K(lYo`JO}gnC zx`V}a7m-hLU9Xvb@K2ymioF)vj12<*^oAqRuG_4u%(ah?+go%$kOpfb`T96P+L$4> zQ#S+sA%VbH&mD1k5Ak7^^dZoC>`1L%i>ZXmooA!%GI)b+$D&ziKrb)a=-ds9xk#~& z7)3iem6I|r5+ZrTRe_W861x8JpD`DDIYZNm{$baw+$)X^Jtjnl0xlBgdnNY}x%5za zkQ8E6T<^$sKBPtL4(1zi_Rd(tVth*3Xs!ulflX+70?gb&jRTnI8l+*Aj9{|d%qLZ+ z>~V9Z;)`8-lds*Zgs~z1?Fg?Po7|FDl(Ce<*c^2=lFQ~ahwh6rqSjtM5+$GT>3WZW zj;u~w9xwAhOc<kF}~`CJ68 z?(S5vNJa;kriPlim33{N5`C{9?NWhzsna_~^|K2k4xz1`xcui*LXL-1#Y}Hi9`Oo!zQ>x-kgAX4LrPz63uZ+?uG*84@PKq-KgQlMNRwz=6Yes) zY}>YN+qP}nwr$(CZQFjUOI=-6J$2^XGvC~EZ+vrqWaOXB$k?%Suf5k=4>AveC1aJ! ziaW4IS%F$_Babi)kA8Y&u4F7E%99OPtm=vzw$$ zEz#9rvn`Iot_z-r3MtV>k)YvErZ<^Oa${`2>MYYODSr6?QZu+be-~MBjwPGdMvGd!b!elsdi4% z`37W*8+OGulab8YM?`KjJ8e+jM(tqLKSS@=jimq3)Ea2EB%88L8CaM+aG7;27b?5` z4zuUWBr)f)k2o&xg{iZ$IQkJ+SK>lpq4GEacu~eOW4yNFLU!Kgc{w4&D$4ecm0f}~ zTTzquRW@`f0}|IILl`!1P+;69g^upiPA6F{)U8)muWHzexRenBU$E^9X-uIY2%&1w z_=#5*(nmxJ9zF%styBwivi)?#KMG96-H@hD-H_&EZiRNsfk7mjBq{L%!E;Sqn!mVX*}kXhwH6eh;b42eD!*~upVG@ z#smUqz$ICm!Y8wY53gJeS|Iuard0=;k5i5Z_hSIs6tr)R4n*r*rE`>38Pw&lkv{_r!jNN=;#?WbMj|l>cU(9trCq; z%nN~r^y7!kH^GPOf3R}?dDhO=v^3BeP5hF|%4GNQYBSwz;x({21i4OQY->1G=KFyu z&6d`f2tT9Yl_Z8YACZaJ#v#-(gcyeqXMhYGXb=t>)M@fFa8tHp2x;ODX=Ap@a5I=U z0G80^$N0G4=U(>W%mrrThl0DjyQ-_I>+1Tdd_AuB3qpYAqY54upwa3}owa|x5iQ^1 zEf|iTZxKNGRpI>34EwkIQ2zHDEZ=(J@lRaOH>F|2Z%V_t56Km$PUYu^xA5#5Uj4I4RGqHD56xT%H{+P8Ag>e_3pN$4m8n>i%OyJFPNWaEnJ4McUZPa1QmOh?t8~n& z&RulPCors8wUaqMHECG=IhB(-tU2XvHP6#NrLVyKG%Ee*mQ5Ps%wW?mcnriTVRc4J`2YVM>$ixSF2Xi+Wn(RUZnV?mJ?GRdw%lhZ+t&3s7g!~g{%m&i<6 z5{ib-<==DYG93I(yhyv4jp*y3#*WNuDUf6`vTM%c&hiayf(%=x@4$kJ!W4MtYcE#1 zHM?3xw63;L%x3drtd?jot!8u3qeqctceX3m;tWetK+>~q7Be$h>n6riK(5@ujLgRS zvOym)k+VAtyV^mF)$29Y`nw&ijdg~jYpkx%*^ z8dz`C*g=I?;clyi5|!27e2AuSa$&%UyR(J3W!A=ZgHF9OuKA34I-1U~pyD!KuRkjA zbkN!?MfQOeN>DUPBxoy5IX}@vw`EEB->q!)8fRl_mqUVuRu|C@KD-;yl=yKc=ZT0% zB$fMwcC|HE*0f8+PVlWHi>M`zfsA(NQFET?LrM^pPcw`cK+Mo0%8*x8@65=CS_^$cG{GZQ#xv($7J z??R$P)nPLodI;P!IC3eEYEHh7TV@opr#*)6A-;EU2XuogHvC;;k1aI8asq7ovoP!* z?x%UoPrZjj<&&aWpsbr>J$Er-7!E(BmOyEv!-mbGQGeJm-U2J>74>o5x`1l;)+P&~ z>}f^=Rx(ZQ2bm+YE0u=ZYrAV@apyt=v1wb?R@`i_g64YyAwcOUl=C!i>=Lzb$`tjv zOO-P#A+)t-JbbotGMT}arNhJmmGl-lyUpMn=2UacVZxmiG!s!6H39@~&uVokS zG=5qWhfW-WOI9g4!R$n7!|ViL!|v3G?GN6HR0Pt_L5*>D#FEj5wM1DScz4Jv@Sxnl zB@MPPmdI{(2D?;*wd>3#tjAirmUnQoZrVv`xM3hARuJksF(Q)wd4P$88fGYOT1p6U z`AHSN!`St}}UMBT9o7i|G`r$ zrB=s$qV3d6$W9@?L!pl0lf%)xs%1ko^=QY$ty-57=55PvP(^6E7cc zGJ*>m2=;fOj?F~yBf@K@9qwX0hA803Xw+b0m}+#a(>RyR8}*Y<4b+kpp|OS+!whP( zH`v{%s>jsQI9rd$*vm)EkwOm#W_-rLTHcZRek)>AtF+~<(did)*oR1|&~1|e36d-d zgtm5cv1O0oqgWC%Et@P4Vhm}Ndl(Y#C^MD03g#PH-TFy+7!Osv1z^UWS9@%JhswEq~6kSr2DITo59+; ze=ZC}i2Q?CJ~Iyu?vn|=9iKV>4j8KbxhE4&!@SQ^dVa-gK@YfS9xT(0kpW*EDjYUkoj! zE49{7H&E}k%5(>sM4uGY)Q*&3>{aitqdNnRJkbOmD5Mp5rv-hxzOn80QsG=HJ_atI-EaP69cacR)Uvh{G5dTpYG7d zbtmRMq@Sexey)||UpnZ?;g_KMZq4IDCy5}@u!5&B^-=6yyY{}e4Hh3ee!ZWtL*s?G zxG(A!<9o!CL+q?u_utltPMk+hn?N2@?}xU0KlYg?Jco{Yf@|mSGC<(Zj^yHCvhmyx z?OxOYoxbptDK()tsJ42VzXdINAMWL$0Gcw?G(g8TMB)Khw_|v9`_ql#pRd2i*?CZl z7k1b!jQB=9-V@h%;Cnl7EKi;Y^&NhU0mWEcj8B|3L30Ku#-9389Q+(Yet0r$F=+3p z6AKOMAIi|OHyzlHZtOm73}|ntKtFaXF2Fy|M!gOh^L4^62kGUoWS1i{9gsds_GWBc zLw|TaLP64z3z9?=R2|T6Xh2W4_F*$cq>MtXMOy&=IPIJ`;!Tw?PqvI2b*U1)25^<2 zU_ZPoxg_V0tngA0J+mm?3;OYw{i2Zb4x}NedZug!>EoN3DC{1i)Z{Z4m*(y{ov2%- zk(w>+scOO}MN!exSc`TN)!B=NUX`zThWO~M*ohqq;J2hx9h9}|s#?@eR!=F{QTrq~ zTcY|>azkCe$|Q0XFUdpFT=lTcyW##i;-e{}ORB4D?t@SfqGo_cS z->?^rh$<&n9DL!CF+h?LMZRi)qju!meugvxX*&jfD!^1XB3?E?HnwHP8$;uX{Rvp# zh|)hM>XDv$ZGg=$1{+_bA~u-vXqlw6NH=nkpyWE0u}LQjF-3NhATL@9rRxMnpO%f7 z)EhZf{PF|mKIMFxnC?*78(}{Y)}iztV12}_OXffJ;ta!fcFIVjdchyHxH=t%ci`Xd zX2AUB?%?poD6Zv*&BA!6c5S#|xn~DK01#XvjT!w!;&`lDXSJT4_j$}!qSPrb37vc{ z9^NfC%QvPu@vlxaZ;mIbn-VHA6miwi8qJ~V;pTZkKqqOii<1Cs}0i?uUIss;hM4dKq^1O35y?Yp=l4i zf{M!@QHH~rJ&X~8uATV><23zZUbs-J^3}$IvV_ANLS08>k`Td7aU_S1sLsfi*C-m1 z-e#S%UGs4E!;CeBT@9}aaI)qR-6NU@kvS#0r`g&UWg?fC7|b^_HyCE!8}nyh^~o@< zpm7PDFs9yxp+byMS(JWm$NeL?DNrMCNE!I^ko-*csB+dsf4GAq{=6sfyf4wb>?v1v zmb`F*bN1KUx-`ra1+TJ37bXNP%`-Fd`vVQFTwWpX@;s(%nDQa#oWhgk#mYlY*!d>( zE&!|ySF!mIyfING+#%RDY3IBH_fW$}6~1%!G`suHub1kP@&DoAd5~7J55;5_noPI6eLf{t;@9Kf<{aO0`1WNKd?<)C-|?C?)3s z>wEq@8=I$Wc~Mt$o;g++5qR+(6wt9GI~pyrDJ%c?gPZe)owvy^J2S=+M^ z&WhIE`g;;J^xQLVeCtf7b%Dg#Z2gq9hp_%g)-%_`y*zb; zn9`f`mUPN-Ts&fFo(aNTsXPA|J!TJ{0hZp0^;MYHLOcD=r_~~^ymS8KLCSeU3;^QzJNqS z5{5rEAv#l(X?bvwxpU;2%pQftF`YFgrD1jt2^~Mt^~G>T*}A$yZc@(k9orlCGv&|1 zWWvVgiJsCAtamuAYT~nzs?TQFt<1LSEx!@e0~@yd6$b5!Zm(FpBl;(Cn>2vF?k zOm#TTjFwd2D-CyA!mqR^?#Uwm{NBemP>(pHmM}9;;8`c&+_o3#E5m)JzfwN?(f-a4 zyd%xZc^oQx3XT?vcCqCX&Qrk~nu;fxs@JUoyVoi5fqpi&bUhQ2y!Ok2pzsFR(M(|U zw3E+kH_zmTRQ9dUMZWRE%Zakiwc+lgv7Z%|YO9YxAy`y28`Aw;WU6HXBgU7fl@dnt z-fFBV)}H-gqP!1;V@Je$WcbYre|dRdp{xt!7sL3Eoa%IA`5CAA%;Wq8PktwPdULo! z8!sB}Qt8#jH9Sh}QiUtEPZ6H0b*7qEKGJ%ITZ|vH)5Q^2m<7o3#Z>AKc%z7_u`rXA zqrCy{-{8;9>dfllLu$^M5L z-hXs))h*qz%~ActwkIA(qOVBZl2v4lwbM>9l70Y`+T*elINFqt#>OaVWoja8RMsep z6Or3f=oBnA3vDbn*+HNZP?8LsH2MY)x%c13@(XfuGR}R?Nu<|07{$+Lc3$Uv^I!MQ z>6qWgd-=aG2Y^24g4{Bw9ueOR)(9h`scImD=86dD+MnSN4$6 z^U*o_mE-6Rk~Dp!ANp#5RE9n*LG(Vg`1)g6!(XtDzsov$Dvz|Gv1WU68J$CkshQhS zCrc|cdkW~UK}5NeaWj^F4MSgFM+@fJd{|LLM)}_O<{rj z+?*Lm?owq?IzC%U%9EBga~h-cJbIu=#C}XuWN>OLrc%M@Gu~kFEYUi4EC6l#PR2JS zQUkGKrrS#6H7}2l0F@S11DP`@pih0WRkRJl#F;u{c&ZC{^$Z+_*lB)r)-bPgRFE;* zl)@hK4`tEP=P=il02x7-C7p%l=B`vkYjw?YhdJU9!P!jcmY$OtC^12w?vy3<<=tlY zUwHJ_0lgWN9vf>1%WACBD{UT)1qHQSE2%z|JHvP{#INr13jM}oYv_5#xsnv9`)UAO zuwgyV4YZ;O)eSc3(mka6=aRohi!HH@I#xq7kng?Acdg7S4vDJb6cI5fw?2z%3yR+| zU5v@Hm}vy;${cBp&@D=HQ9j7NcFaOYL zj-wV=eYF{|XTkFNM2uz&T8uH~;)^Zo!=KP)EVyH6s9l1~4m}N%XzPpduPg|h-&lL` zAXspR0YMOKd2yO)eMFFJ4?sQ&!`dF&!|niH*!^*Ml##o0M(0*uK9&yzekFi$+mP9s z>W9d%Jb)PtVi&-Ha!o~Iyh@KRuKpQ@)I~L*d`{O8!kRObjO7=n+Gp36fe!66neh+7 zW*l^0tTKjLLzr`x4`_8&on?mjW-PzheTNox8Hg7Nt@*SbE-%kP2hWYmHu#Fn@Q^J(SsPUz*|EgOoZ6byg3ew88UGdZ>9B2Tq=jF72ZaR=4u%1A6Vm{O#?@dD!(#tmR;eP(Fu z{$0O%=Vmua7=Gjr8nY%>ul?w=FJ76O2js&17W_iq2*tb!i{pt#`qZB#im9Rl>?t?0c zicIC}et_4d+CpVPx)i4~$u6N-QX3H77ez z?ZdvXifFk|*F8~L(W$OWM~r`pSk5}#F?j_5u$Obu9lDWIknO^AGu+Blk7!9Sb;NjS zncZA?qtASdNtzQ>z7N871IsPAk^CC?iIL}+{K|F@BuG2>qQ;_RUYV#>hHO(HUPpk@ z(bn~4|F_jiZi}Sad;_7`#4}EmD<1EiIxa48QjUuR?rC}^HRocq`OQPM@aHVKP9E#q zy%6bmHygCpIddPjE}q_DPC`VH_2m;Eey&ZH)E6xGeStOK7H)#+9y!%-Hm|QF6w#A( zIC0Yw%9j$s-#odxG~C*^MZ?M<+&WJ+@?B_QPUyTg9DJGtQN#NIC&-XddRsf3n^AL6 zT@P|H;PvN;ZpL0iv$bRb7|J{0o!Hq+S>_NrH4@coZtBJu#g8#CbR7|#?6uxi8d+$g z87apN>EciJZ`%Zv2**_uiET9Vk{pny&My;+WfGDw4EVL#B!Wiw&M|A8f1A@ z(yFQS6jfbH{b8Z-S7D2?Ixl`j0{+ZnpT=;KzVMLW{B$`N?Gw^Fl0H6lT61%T2AU**!sX0u?|I(yoy&Xveg7XBL&+>n6jd1##6d>TxE*Vj=8lWiG$4=u{1UbAa5QD>5_ z;Te^42v7K6Mmu4IWT6Rnm>oxrl~b<~^e3vbj-GCdHLIB_>59}Ya+~OF68NiH=?}2o zP(X7EN=quQn&)fK>M&kqF|<_*H`}c zk=+x)GU>{Af#vx&s?`UKUsz})g^Pc&?Ka@t5$n$bqf6{r1>#mWx6Ep>9|A}VmWRnowVo`OyCr^fHsf# zQjQ3Ttp7y#iQY8l`zEUW)(@gGQdt(~rkxlkefskT(t%@i8=|p1Y9Dc5bc+z#n$s13 zGJk|V0+&Ekh(F};PJzQKKo+FG@KV8a<$gmNSD;7rd_nRdc%?9)p!|B-@P~kxQG}~B zi|{0}@}zKC(rlFUYp*dO1RuvPC^DQOkX4<+EwvBAC{IZQdYxoq1Za!MW7%p7gGr=j zzWnAq%)^O2$eItftC#TTSArUyL$U54-O7e|)4_7%Q^2tZ^0-d&3J1}qCzR4dWX!)4 zzIEKjgnYgMus^>6uw4Jm8ga6>GBtMjpNRJ6CP~W=37~||gMo_p@GA@#-3)+cVYnU> zE5=Y4kzl+EbEh%dhQokB{gqNDqx%5*qBusWV%!iprn$S!;oN_6E3?0+umADVs4ako z?P+t?m?};gev9JXQ#Q&KBpzkHPde_CGu-y z<{}RRAx=xlv#mVi+Ibrgx~ujW$h{?zPfhz)Kp7kmYS&_|97b&H&1;J-mzrBWAvY} zh8-I8hl_RK2+nnf&}!W0P+>5?#?7>npshe<1~&l_xqKd0_>dl_^RMRq@-Myz&|TKZBj1=Q()) zF{dBjv5)h=&Z)Aevx}+i|7=R9rG^Di!sa)sZCl&ctX4&LScQ-kMncgO(9o6W6)yd< z@Rk!vkja*X_N3H=BavGoR0@u0<}m-7|2v!0+2h~S2Q&a=lTH91OJsvms2MT~ zY=c@LO5i`mLpBd(vh|)I&^A3TQLtr>w=zoyzTd=^f@TPu&+*2MtqE$Avf>l>}V|3-8Fp2hzo3y<)hr_|NO(&oSD z!vEjTWBxbKTiShVl-U{n*B3#)3a8$`{~Pk}J@elZ=>Pqp|MQ}jrGv7KrNcjW%TN_< zZz8kG{#}XoeWf7qY?D)L)8?Q-b@Na&>i=)(@uNo zr;cH98T3$Iau8Hn*@vXi{A@YehxDE2zX~o+RY`)6-X{8~hMpc#C`|8y> zU8Mnv5A0dNCf{Ims*|l-^ z(MRp{qoGohB34|ggDI*p!Aw|MFyJ|v+<+E3brfrI)|+l3W~CQLPbnF@G0)P~Ly!1TJLp}xh8uW`Q+RB-v`MRYZ9Gam3cM%{ zb4Cb*f)0deR~wtNb*8w-LlIF>kc7DAv>T0D(a3@l`k4TFnrO+g9XH7;nYOHxjc4lq zMmaW6qpgAgy)MckYMhl?>sq;-1E)-1llUneeA!ya9KM$)DaNGu57Z5aE>=VST$#vb zFo=uRHr$0M{-ha>h(D_boS4zId;3B|Tpqo|?B?Z@I?G(?&Iei+-{9L_A9=h=Qfn-U z1wIUnQe9!z%_j$F_{rf&`ZFSott09gY~qrf@g3O=Y>vzAnXCyL!@(BqWa)Zqt!#_k zfZHuwS52|&&)aK;CHq9V-t9qt0au{$#6c*R#e5n3rje0hic7c7m{kW$p(_`wB=Gw7 z4k`1Hi;Mc@yA7dp@r~?@rfw)TkjAW++|pkfOG}0N|2guek}j8Zen(!+@7?qt_7ndX zB=BG6WJ31#F3#Vk3=aQr8T)3`{=p9nBHlKzE0I@v`{vJ}h8pd6vby&VgFhzH|q;=aonunAXL6G2y(X^CtAhWr*jI zGjpY@raZDQkg*aMq}Ni6cRF z{oWv}5`nhSAv>usX}m^GHt`f(t8@zHc?K|y5Zi=4G*UG1Sza{$Dpj%X8 zzEXaKT5N6F5j4J|w#qlZP!zS7BT)9b+!ZSJdToqJts1c!)fwih4d31vfb{}W)EgcA zH2pZ^8_k$9+WD2n`6q5XbOy8>3pcYH9 z07eUB+p}YD@AH!}p!iKv><2QF-Y^&xx^PAc1F13A{nUeCDg&{hnix#FiO!fe(^&%Qcux!h znu*S!s$&nnkeotYsDthh1dq(iQrE|#f_=xVgfiiL&-5eAcC-> z5L0l|DVEM$#ulf{bj+Y~7iD)j<~O8CYM8GW)dQGq)!mck)FqoL^X zwNdZb3->hFrbHFm?hLvut-*uK?zXn3q1z|UX{RZ;-WiLoOjnle!xs+W0-8D)kjU#R z+S|A^HkRg$Ij%N4v~k`jyHffKaC~=wg=9)V5h=|kLQ@;^W!o2^K+xG&2n`XCd>OY5Ydi= zgHH=lgy++erK8&+YeTl7VNyVm9-GfONlSlVb3)V9NW5tT!cJ8d7X)!b-$fb!s76{t z@d=Vg-5K_sqHA@Zx-L_}wVnc@L@GL9_K~Zl(h5@AR#FAiKad8~KeWCo@mgXIQ#~u{ zgYFwNz}2b6Vu@CP0XoqJ+dm8px(5W5-Jpis97F`+KM)TuP*X8H@zwiVKDKGVp59pI zifNHZr|B+PG|7|Y<*tqap0CvG7tbR1R>jn70t1X`XJixiMVcHf%Ez*=xm1(CrTSDt z0cle!+{8*Ja&EOZ4@$qhBuKQ$U95Q%rc7tg$VRhk?3=pE&n+T3upZg^ZJc9~c2es% zh7>+|mrmA-p&v}|OtxqmHIBgUxL~^0+cpfkSK2mhh+4b=^F1Xgd2)}U*Yp+H?ls#z zrLxWg_hm}AfK2XYWr!rzW4g;+^^&bW%LmbtRai9f3PjU${r@n`JThy-cphbcwn)rq9{A$Ht`lmYKxOacy z6v2R(?gHhD5@&kB-Eg?4!hAoD7~(h>(R!s1c1Hx#s9vGPePUR|of32bS`J5U5w{F) z>0<^ktO2UHg<0{oxkdOQ;}coZDQph8p6ruj*_?uqURCMTac;>T#v+l1Tc~%^k-Vd@ zkc5y35jVNc49vZpZx;gG$h{%yslDI%Lqga1&&;mN{Ush1c7p>7e-(zp}6E7f-XmJb4nhk zb8zS+{IVbL$QVF8pf8}~kQ|dHJAEATmmnrb_wLG}-yHe>W|A&Y|;muy-d^t^<&)g5SJfaTH@P1%euONny=mxo+C z4N&w#biWY41r8k~468tvuYVh&XN&d#%QtIf9;iVXfWY)#j=l`&B~lqDT@28+Y!0E+MkfC}}H*#(WKKdJJq=O$vNYCb(ZG@p{fJgu;h z21oHQ(14?LeT>n5)s;uD@5&ohU!@wX8w*lB6i@GEH0pM>YTG+RAIWZD;4#F1&F%Jp zXZUml2sH0!lYJT?&sA!qwez6cXzJEd(1ZC~kT5kZSp7(@=H2$Azb_*W&6aA|9iwCL zdX7Q=42;@dspHDwYE?miGX#L^3xD&%BI&fN9^;`v4OjQXPBaBmOF1;#C)8XA(WFlH zycro;DS2?(G&6wkr6rqC>rqDv3nfGw3hmN_9Al>TgvmGsL8_hXx09};l9Ow@)F5@y z#VH5WigLDwZE4nh^7&@g{1FV^UZ%_LJ-s<{HN*2R$OPg@R~Z`c-ET*2}XB@9xvAjrK&hS=f|R8Gr9 zr|0TGOsI7RD+4+2{ZiwdVD@2zmg~g@^D--YL;6UYGSM8i$NbQr4!c7T9rg!8;TM0E zT#@?&S=t>GQm)*ua|?TLT2ktj#`|R<_*FAkOu2Pz$wEc%-=Y9V*$&dg+wIei3b*O8 z2|m$!jJG!J!ZGbbIa!(Af~oSyZV+~M1qGvelMzPNE_%5?c2>;MeeG2^N?JDKjFYCy z7SbPWH-$cWF9~fX%9~v99L!G(wi!PFp>rB!9xj7=Cv|F+7CsGNwY0Q_J%FID%C^CBZQfJ9K(HK%k31j~e#&?hQ zNuD6gRkVckU)v+53-fc} z7ZCzYN-5RG4H7;>>Hg?LU9&5_aua?A0)0dpew1#MMlu)LHe(M;OHjHIUl7|%%)YPo z0cBk;AOY00%Fe6heoN*$(b<)Cd#^8Iu;-2v@>cE-OB$icUF9EEoaC&q8z9}jMTT2I z8`9;jT%z0;dy4!8U;GW{i`)3!c6&oWY`J3669C!tM<5nQFFrFRglU8f)5Op$GtR-3 zn!+SPCw|04sv?%YZ(a7#L?vsdr7ss@WKAw&A*}-1S|9~cL%uA+E~>N6QklFE>8W|% zyX-qAUGTY1hQ-+um`2|&ji0cY*(qN!zp{YpDO-r>jPk*yuVSay<)cUt`t@&FPF_&$ zcHwu1(SQ`I-l8~vYyUxm@D1UEdFJ$f5Sw^HPH7b!9 zzYT3gKMF((N(v0#4f_jPfVZ=ApN^jQJe-X$`A?X+vWjLn_%31KXE*}5_}d8 zw_B1+a#6T1?>M{ronLbHIlEsMf93muJ7AH5h%;i99<~JX^;EAgEB1uHralD*!aJ@F zV2ruuFe9i2Q1C?^^kmVy921eb=tLDD43@-AgL^rQ3IO9%+vi_&R2^dpr}x{bCVPej z7G0-0o64uyWNtr*loIvslyo0%)KSDDKjfThe0hcqs)(C-MH1>bNGBDRTW~scy_{w} zp^aq8Qb!h9Lwielq%C1b8=?Z=&U)ST&PHbS)8Xzjh2DF?d{iAv)Eh)wsUnf>UtXN( zL7=$%YrZ#|^c{MYmhn!zV#t*(jdmYdCpwqpZ{v&L8KIuKn`@IIZfp!uo}c;7J57N` zAxyZ-uA4=Gzl~Ovycz%MW9ZL7N+nRo&1cfNn9(1H5eM;V_4Z_qVann7F>5f>%{rf= zPBZFaV@_Sobl?Fy&KXyzFDV*FIdhS5`Uc~S^Gjo)aiTHgn#<0C=9o-a-}@}xDor;D zZyZ|fvf;+=3MZd>SR1F^F`RJEZo+|MdyJYQAEauKu%WDol~ayrGU3zzbHKsnHKZ*z zFiwUkL@DZ>!*x05ql&EBq@_Vqv83&?@~q5?lVmffQZ+V-=qL+!u4Xs2Z2zdCQ3U7B&QR9_Iggy} z(om{Y9eU;IPe`+p1ifLx-XWh?wI)xU9ik+m#g&pGdB5Bi<`PR*?92lE0+TkRuXI)z z5LP!N2+tTc%cB6B1F-!fj#}>S!vnpgVU~3!*U1ej^)vjUH4s-bd^%B=ItQqDCGbrEzNQi(dJ`J}-U=2{7-d zK8k^Rlq2N#0G?9&1?HSle2vlkj^KWSBYTwx`2?9TU_DX#J+f+qLiZCqY1TXHFxXZqYMuD@RU$TgcnCC{_(vwZ-*uX)~go#%PK z@}2Km_5aQ~(<3cXeJN6|F8X_1@L%@xTzs}$_*E|a^_URF_qcF;Pfhoe?FTFwvjm1o z8onf@OY@jC2tVcMaZS;|T!Ks(wOgPpRzRnFS-^RZ4E!9dsnj9sFt609a|jJbb1Dt@ z<=Gal2jDEupxUSwWu6zp<<&RnAA;d&4gKVG0iu6g(DsST(4)z6R)zDpfaQ}v{5ARt zyhwvMtF%b-YazR5XLz+oh=mn;y-Mf2a8>7?2v8qX;19y?b>Z5laGHvzH;Nu9S`B8} zI)qN$GbXIQ1VL3lnof^6TS~rvPVg4V?Dl2Bb*K2z4E{5vy<(@@K_cN@U>R!>aUIRnb zL*)=787*cs#zb31zBC49x$`=fkQbMAef)L2$dR{)6BAz!t5U_B#1zZG`^neKSS22oJ#5B=gl%U=WeqL9REF2g zZnfCb0?quf?Ztj$VXvDSWoK`0L=Zxem2q}!XWLoT-kYMOx)!7fcgT35uC~0pySEme z`{wGWTkGr7>+Kb^n;W?BZH6ZP(9tQX%-7zF>vc2}LuWDI(9kh1G#7B99r4x6;_-V+k&c{nPUrR zAXJGRiMe~aup{0qzmLNjS_BC4cB#sXjckx{%_c&^xy{M61xEb>KW_AG5VFXUOjAG4 z^>Qlm9A#1N{4snY=(AmWzatb!ngqiqPbBZ7>Uhb3)dTkSGcL#&SH>iMO-IJBPua`u zo)LWZ>=NZLr758j{%(|uQuZ)pXq_4c!!>s|aDM9#`~1bzK3J1^^D#<2bNCccH7~-X}Ggi!pIIF>uFx%aPARGQsnC8ZQc8lrQ5o~smqOg>Ti^GNme94*w z)JZy{_{#$jxGQ&`M z!OMvZMHR>8*^>eS%o*6hJwn!l8VOOjZQJvh)@tnHVW&*GYPuxqXw}%M!(f-SQf`=L z5;=5w2;%82VMH6Xi&-K3W)o&K^+vJCepWZ-rW%+Dc6X3(){z$@4zjYxQ|}8UIojeC zYZpQ1dU{fy=oTr<4VX?$q)LP}IUmpiez^O&N3E_qPpchGTi5ZM6-2ScWlQq%V&R2Euz zO|Q0Hx>lY1Q1cW5xHv5!0OGU~PVEqSuy#fD72d#O`N!C;o=m+YioGu-wH2k6!t<~K zSr`E=W9)!g==~x9VV~-8{4ZN9{~-A9zJpRe%NGg$+MDuI-dH|b@BD)~>pPCGUNNzY zMDg||0@XGQgw`YCt5C&A{_+J}mvV9Wg{6V%2n#YSRN{AP#PY?1FF1#|vO_%e+#`|2*~wGAJaeRX6=IzFNeWhz6gJc8+(03Ph4y6ELAm=AkN7TOgMUEw*N{= z_)EIDQx5q22oUR+_b*tazu9+pX|n1c*IB-}{DqIj z-?E|ks{o3AGRNb;+iKcHkZvYJvFsW&83RAPs1Oh@IWy%l#5x2oUP6ZCtv+b|q>jsf zZ_9XO;V!>n`UxH1LvH8)L4?8raIvasEhkpQoJ`%!5rBs!0Tu(s_D{`4opB;57)pkX z4$A^8CsD3U5*!|bHIEqsn~{q+Ddj$ME@Gq4JXtgVz&7l{Ok!@?EA{B3P~NAqb9)4? zkQo30A^EbHfQ@87G5&EQTd`frrwL)&Yw?%-W@uy^Gn23%j?Y!Iea2xw<-f;esq zf%w5WN@E1}zyXtYv}}`U^B>W`>XPmdLj%4{P298|SisrE;7HvXX;A}Ffi8B#3Lr;1 zHt6zVb`8{#+e$*k?w8|O{Uh|&AG}|DG1PFo1i?Y*cQm$ZwtGcVgMwtBUDa{~L1KT-{jET4w60>{KZ27vXrHJ;fW{6| z=|Y4!&UX020wU1>1iRgB@Q#m~1^Z^9CG1LqDhYBrnx%IEdIty z!46iOoKlKs)c}newDG)rWUikD%j`)p z_w9Ph&e40=(2eBy;T!}*1p1f1SAUDP9iWy^u^Ubdj21Kn{46;GR+hwLO=4D11@c~V zI8x&(D({K~Df2E)Nx_yQvYfh4;MbMJ@Z}=Dt3_>iim~QZ*hZIlEs0mEb z_54+&*?wMD`2#vsQRN3KvoT>hWofI_Vf(^C1ff-Ike@h@saEf7g}<9T`W;HAne-Nd z>RR+&SP35w)xKn8^U$7))PsM!jKwYZ*RzEcG-OlTrX3}9a{q%#Un5E5W{{hp>w~;` zGky+3(vJvQyGwBo`tCpmo0mo((?nM8vf9aXrrY1Ve}~TuVkB(zeds^jEfI}xGBCM2 zL1|#tycSaWCurP+0MiActG3LCas@_@tao@(R1ANlwB$4K53egNE_;!&(%@Qo$>h`^1S_!hN6 z)vZtG$8fN!|BXBJ=SI>e(LAU(y(i*PHvgQ2llulxS8>qsimv7yL}0q_E5WiAz7)(f zC(ahFvG8&HN9+6^jGyLHM~$)7auppeWh_^zKk&C_MQ~8;N??OlyH~azgz5fe^>~7F zl3HnPN3z-kN)I$4@`CLCMQx3sG~V8hPS^}XDXZrQA>}mQPw%7&!sd(Pp^P=tgp-s^ zjl}1-KRPNWXgV_K^HkP__SR`S-|OF0bR-N5>I%ODj&1JUeAQ3$9i;B~$S6}*^tK?= z**%aCiH7y?xdY?{LgVP}S0HOh%0%LI$wRx;$T|~Y8R)Vdwa}kGWv8?SJVm^>r6+%I z#lj1aR94{@MP;t-scEYQWc#xFA30^}?|BeX*W#9OL;Q9#WqaaM546j5j29((^_8Nu z4uq}ESLr~r*O7E7$D{!k9W>`!SLoyA53i9QwRB{!pHe8um|aDE`Cg0O*{jmor)^t)3`>V>SWN-2VJcFmj^1?~tT=JrP`fVh*t zXHarp=8HEcR#vFe+1a%XXuK+)oFs`GDD}#Z+TJ}Ri`FvKO@ek2ayn}yaOi%(8p%2$ zpEu)v0Jym@f}U|-;}CbR=9{#<^z28PzkkTNvyKvJDZe+^VS2bES3N@Jq!-*}{oQlz z@8bgC_KnDnT4}d#&Cpr!%Yb?E!brx0!eVOw~;lLwUoz#Np%d$o%9scc3&zPm`%G((Le|6o1 zM(VhOw)!f84zG^)tZ1?Egv)d8cdNi+T${=5kV+j;Wf%2{3g@FHp^Gf*qO0q!u$=m9 zCaY`4mRqJ;FTH5`a$affE5dJrk~k`HTP_7nGTY@B9o9vvnbytaID;^b=Tzp7Q#DmD zC(XEN)Ktn39z5|G!wsVNnHi) z%^q94!lL|hF`IijA^9NR0F$@h7k5R^ljOW(;Td9grRN0Mb)l_l7##{2nPQ@?;VjXv zaLZG}yuf$r$<79rVPpXg?6iiieX|r#&`p#Con2i%S8*8F}(E) zI5E6c3tG*<;m~6>!&H!GJ6zEuhH7mkAzovdhLy;)q z{H2*8I^Pb}xC4s^6Y}6bJvMu=8>g&I)7!N!5QG$xseeU#CC?ZM-TbjsHwHgDGrsD= z{%f;@Sod+Ch66Ko2WF~;Ty)v>&x^aovCbCbD7>qF*!?BXmOV3(s|nxsb*Lx_2lpB7 zokUnzrk;P=T-&kUHO}td+Zdj!3n&NR?K~cRU zAXU!DCp?51{J4w^`cV#ye}(`SQhGQkkMu}O3M*BWt4UsC^jCFUy;wTINYmhD$AT;4 z?Xd{HaJjP`raZ39qAm;%beDbrLpbRf(mkKbANan7XsL>_pE2oo^$TgdidjRP!5-`% zv0d!|iKN$c0(T|L0C~XD0aS8t{*&#LnhE;1Kb<9&=c2B+9JeLvJr*AyyRh%@jHej=AetOMSlz^=!kxX>>B{2B1uIrQyfd8KjJ+DBy!h)~*(!|&L4^Q_07SQ~E zcemVP`{9CwFvPFu7pyVGCLhH?LhEVb2{7U+Z_>o25#+3<|8%1T^5dh}*4(kfJGry} zm%r#hU+__Z;;*4fMrX=Bkc@7|v^*B;HAl0((IBPPii%X9+u3DDF6%bI&6?Eu$8&aWVqHIM7mK6?Uvq$1|(-T|)IV<>e?!(rY zqkmO1MRaLeTR=)io(0GVtQT@s6rN%C6;nS3@eu;P#ry4q;^O@1ZKCJyp_Jo)Ty^QW z+vweTx_DLm{P-XSBj~Sl<%_b^$=}odJ!S2wAcxenmzFGX1t&Qp8Vxz2VT`uQsQYtdn&_0xVivIcxZ_hnrRtwq4cZSj1c-SG9 z7vHBCA=fd0O1<4*=lu$6pn~_pVKyL@ztw1swbZi0B?spLo56ZKu5;7ZeUml1Ws1?u zqMf1p{5myAzeX$lAi{jIUqo1g4!zWLMm9cfWcnw`k6*BR^?$2(&yW?>w;G$EmTA@a z6?y#K$C~ZT8+v{87n5Dm&H6Pb_EQ@V0IWmG9cG=O;(;5aMWWrIPzz4Q`mhK;qQp~a z+BbQrEQ+w{SeiuG-~Po5f=^EvlouB@_|4xQXH@A~KgpFHrwu%dwuCR)=B&C(y6J4J zvoGk9;lLs9%iA-IJGU#RgnZZR+@{5lYl8(e1h6&>Vc_mvg0d@);X zji4T|n#lB!>pfL|8tQYkw?U2bD`W{na&;*|znjmalA&f;*U++_aBYerq;&C8Kw7mI z7tsG*?7*5j&dU)Lje;^{D_h`%(dK|pB*A*1(Jj)w^mZ9HB|vGLkF1GEFhu&rH=r=8 zMxO42e{Si6$m+Zj`_mXb&w5Q(i|Yxyg?juUrY}78uo@~3v84|8dfgbPd0iQJRdMj< zncCNGdMEcsxu#o#B5+XD{tsg*;j-eF8`mp~K8O1J!Z0+>0=7O=4M}E?)H)ENE;P*F z$Ox?ril_^p0g7xhDUf(q652l|562VFlC8^r8?lQv;TMvn+*8I}&+hIQYh2 z1}uQQaag&!-+DZ@|C+C$bN6W;S-Z@)d1|en+XGvjbOxCa-qAF*LA=6s(Jg+g;82f$ z(Vb)8I)AH@cdjGFAR5Rqd0wiNCu!xtqWbcTx&5kslzTb^7A78~Xzw1($UV6S^VWiP zFd{Rimd-0CZC_Bu(WxBFW7+k{cOW7DxBBkJdJ;VsJ4Z@lERQr%3eVv&$%)b%<~ zCl^Y4NgO}js@u{|o~KTgH}>!* z_iDNqX2(As7T0xivMH|3SC1ivm8Q}6Ffcd7owUKN5lHAtzMM4<0v+ykUT!QiowO;`@%JGv+K$bBx@*S7C8GJVqQ_K>12}M`f_Ys=S zKFh}HM9#6Izb$Y{wYzItTy+l5U2oL%boCJn?R3?jP@n$zSIwlmyGq30Cw4QBO|14` zW5c);AN*J3&eMFAk$SR~2k|&+&Bc$e>s%c{`?d~85S-UWjA>DS5+;UKZ}5oVa5O(N zqqc@>)nee)+4MUjH?FGv%hm2{IlIF-QX}ym-7ok4Z9{V+ZHVZQl$A*x!(q%<2~iVv znUa+BX35&lCb#9VE-~Y^W_f;Xhl%vgjwdjzMy$FsSIj&ok}L+X`4>J=9BkN&nu^E*gbhj3(+D>C4E z@Fwq_=N)^bKFSHTzZk?-gNU$@l}r}dwGyh_fNi=9b|n}J>&;G!lzilbWF4B}BBq4f zYIOl?b)PSh#XTPp4IS5ZR_2C!E)Z`zH0OW%4;&~z7UAyA-X|sh9@~>cQW^COA9hV4 zXcA6qUo9P{bW1_2`eo6%hgbN%(G-F1xTvq!sc?4wN6Q4`e9Hku zFwvlAcRY?6h^Fj$R8zCNEDq8`=uZB8D-xn)tA<^bFFy}4$vA}Xq0jAsv1&5!h!yRA zU()KLJya5MQ`q&LKdH#fwq&(bNFS{sKlEh_{N%{XCGO+po#(+WCLmKW6&5iOHny>g z3*VFN?mx!16V5{zyuMWDVP8U*|BGT$(%IO|)?EF|OI*sq&RovH!N%=>i_c?K*A>>k zyg1+~++zY4Q)J;VWN0axhoIKx;l&G$gvj(#go^pZskEVj8^}is3Jw26LzYYVos0HX zRPvmK$dVxM8(Tc?pHFe0Z3uq){{#OK3i-ra#@+;*=ui8)y6hsRv z4Fxx1c1+fr!VI{L3DFMwXKrfl#Q8hfP@ajgEau&QMCxd{g#!T^;ATXW)nUg&$-n25 zruy3V!!;{?OTobo|0GAxe`Acn3GV@W=&n;~&9 zQM>NWW~R@OYORkJAo+eq1!4vzmf9K%plR4(tB@TR&FSbDoRgJ8qVcH#;7lQub*nq&?Z>7WM=oeEVjkaG zT#f)=o!M2DO5hLR+op>t0CixJCIeXH*+z{-XS|%jx)y(j&}Wo|3!l7{o)HU3m7LYyhv*xF&tq z%IN7N;D4raue&&hm0xM=`qv`+TK@;_xAcGKuK(2|75~ar2Yw)geNLSmVxV@x89bQu zpViVKKnlkwjS&&c|-X6`~xdnh}Ps)Hs z4VbUL^{XNLf7_|Oi>tA%?SG5zax}esF*FH3d(JH^Gvr7Rp*n=t7frH!U;!y1gJB^i zY_M$KL_}mW&XKaDEi9K-wZR|q*L32&m+2n_8lq$xRznJ7p8}V>w+d@?uB!eS3#u<} zIaqi!b!w}a2;_BfUUhGMy#4dPx>)_>yZ`ai?Rk`}d0>~ce-PfY-b?Csd(28yX22L% zI7XI>OjIHYTk_@Xk;Gu^F52^Gn6E1&+?4MxDS2G_#PQ&yXPXP^<-p|2nLTb@AAQEY zI*UQ9Pmm{Kat}wuazpjSyXCdnrD&|C1c5DIb1TnzF}f4KIV6D)CJ!?&l&{T)e4U%3HTSYqsQ zo@zWB1o}ceQSV)<4G<)jM|@@YpL+XHuWsr5AYh^Q{K=wSV99D~4RRU52FufmMBMmd z_H}L#qe(}|I9ZyPRD6kT>Ivj&2Y?qVZq<4bG_co_DP`sE*_Xw8D;+7QR$Uq(rr+u> z8bHUWbV19i#)@@G4bCco@Xb<8u~wVDz9S`#k@ciJtlu@uP1U0X?yov8v9U3VOig2t zL9?n$P3=1U_Emi$#slR>N5wH-=J&T=EdUHA}_Z zZIl3nvMP*AZS9{cDqFanrA~S5BqxtNm9tlu;^`)3X&V4tMAkJ4gEIPl= zoV!Gyx0N{3DpD@)pv^iS*dl2FwANu;1;%EDl}JQ7MbxLMAp>)UwNwe{=V}O-5C*>F zu?Ny+F64jZn<+fKjF01}8h5H_3pey|;%bI;SFg$w8;IC<8l|3#Lz2;mNNik6sVTG3 z+Su^rIE#40C4a-587$U~%KedEEw1%r6wdvoMwpmlXH$xPnNQN#f%Z7|p)nC>WsuO= z4zyqapLS<8(UJ~Qi9d|dQijb_xhA2)v>la)<1md5s^R1N&PiuA$^k|A<+2C?OiHbj z>Bn$~t)>Y(Zb`8hW7q9xQ=s>Rv81V+UiuZJc<23HplI88isqRCId89fb`Kt|CxVIg znWcwprwXnotO>3s&Oypkte^9yJjlUVVxSe%_xlzmje|mYOVPH^vjA=?6xd0vaj0Oz zwJ4OJNiFdnHJX3rw&inskjryukl`*fRQ#SMod5J|KroJRsVXa5_$q7whSQ{gOi*s0 z1LeCy|JBWRsDPn7jCb4s(p|JZiZ8+*ExC@Vj)MF|*Vp{B(ziccSn`G1Br9bV(v!C2 z6#?eqpJBc9o@lJ#^p-`-=`4i&wFe>2)nlPK1p9yPFzJCzBQbpkcR>={YtamIw)3nt z(QEF;+)4`>8^_LU)_Q3 zC5_7lgi_6y>U%m)m@}Ku4C}=l^J=<<7c;99ec3p{aR+v=diuJR7uZi%aQv$oP?dn?@6Yu_+*^>T0ptf(oobdL;6)N-I!TO`zg^Xbv3#L0I~sn@WGk-^SmPh5>W+LB<+1PU}AKa?FCWF|qMNELOgdxR{ zbqE7@jVe+FklzdcD$!(A$&}}H*HQFTJ+AOrJYnhh}Yvta(B zQ_bW4Rr;R~&6PAKwgLWXS{Bnln(vUI+~g#kl{r+_zbngT`Y3`^Qf=!PxN4IYX#iW4 zucW7@LLJA9Zh3(rj~&SyN_pjO8H&)|(v%!BnMWySBJV=eSkB3YSTCyIeJ{i;(oc%_hk{$_l;v>nWSB)oVeg+blh=HB5JSlG_r7@P z3q;aFoZjD_qS@zygYqCn=;Zxjo!?NK!%J$ z52lOP`8G3feEj+HTp@Tnn9X~nG=;tS+z}u{mQX_J0kxtr)O30YD%oo)L@wy`jpQYM z@M>Me=95k1p*FW~rHiV1CIfVc{K8r|#Kt(ApkXKsDG$_>76UGNhHExFCw#Ky9*B-z zNq2ga*xax!HMf_|Vp-86r{;~YgQKqu7%szk8$hpvi_2I`OVbG1doP(`gn}=W<8%Gn z%81#&WjkH4GV;4u43EtSW>K_Ta3Zj!XF?;SO3V#q=<=>Tc^@?A`i;&`-cYj|;^ zEo#Jl5zSr~_V-4}y8pnufXLa80vZY4z2ko7fj>DR)#z=wWuS1$$W!L?(y}YC+yQ|G z@L&`2upy3f>~*IquAjkVNU>}c10(fq#HdbK$~Q3l6|=@-eBbo>B9(6xV`*)sae58*f zym~RRVx;xoCG3`JV`xo z!lFw)=t2Hy)e!IFs?0~7osWk(d%^wxq&>_XD4+U#y&-VF%4z?XH^i4w`TxpF{`XhZ z%G}iEzf!T(l>g;W9<~K+)$g!{UvhW{E0Lis(S^%I8OF&%kr!gJ&fMOpM=&=Aj@wuL zBX?*6i51Qb$uhkwkFYkaD_UDE+)rh1c;(&Y=B$3)J&iJfQSx!1NGgPtK!$c9OtJuu zX(pV$bfuJpRR|K(dp@^j}i&HeJOh@|7lWo8^$*o~Xqo z5Sb+!EtJ&e@6F+h&+_1ETbg7LfP5GZjvIUIN3ibCOldAv z)>YdO|NH$x7AC8dr=<2ekiY1%fN*r~e5h6Yaw<{XIErujKV~tiyrvV_DV0AzEknC- zR^xKM3i<1UkvqBj3C{wDvytOd+YtDSGu!gEMg+!&|8BQrT*|p)(dwQLEy+ zMtMzij3zo40)CA!BKZF~yWg?#lWhqD3@qR)gh~D{uZaJO;{OWV8XZ_)J@r3=)T|kt zUS1pXr6-`!Z}w2QR7nP%d?ecf90;K_7C3d!UZ`N(TZoWNN^Q~RjVhQG{Y<%E1PpV^4 z-m-K+$A~-+VDABs^Q@U*)YvhY4Znn2^w>732H?NRK(5QSS$V@D7yz2BVX4)f5A04~$WbxGOam22>t&uD)JB8-~yiQW6ik;FGblY_I>SvB_z2?PS z*Qm&qbKI{H1V@YGWzpx`!v)WeLT02};JJo*#f$a*FH?IIad-^(;9XC#YTWN6;Z6+S zm4O1KH=#V@FJw7Pha0!9Vb%ZIM$)a`VRMoiN&C|$YA3~ZC*8ayZRY^fyuP6$n%2IU z$#XceYZeqLTXw(m$_z|33I$B4k~NZO>pP6)H_}R{E$i%USGy{l{-jOE;%CloYPEU+ zRFxOn4;7lIOh!7abb23YKD+_-?O z0FP9otcAh+oSj;=f#$&*ExUHpd&e#bSF%#8*&ItcL2H$Sa)?pt0Xtf+t)z$_u^wZi z44oE}r4kIZGy3!Mc8q$B&6JqtnHZ>Znn!Zh@6rgIu|yU+zG8q`q9%B18|T|oN3zMq z`l&D;U!OL~%>vo&q0>Y==~zLiCZk4v%s_7!9DxQ~id1LLE93gf*gg&2$|hB#j8;?3 z5v4S;oM6rT{Y;I+#FdmNw z){d%tNM<<#GN%n9ox7B=3#;u7unZ~tLB_vRZ52a&2=IM)2VkXm=L+Iqq~uk#Dug|x z>S84e+A7EiOY5lj*!q?6HDkNh~0g;0Jy(al!ZHHDtur9T$y-~)94HelX1NHjXWIM7UAe}$?jiz z9?P4`I0JM=G5K{3_%2jPLC^_Mlw?-kYYgb7`qGa3@dn|^1fRMwiyM@Ch z;CB&o7&&?c5e>h`IM;Wnha0QKnEp=$hA8TJgR-07N~U5(>9vJzeoFsSRBkDq=x(YgEMpb=l4TDD`2 zwVJpWGTA_u7}?ecW7s6%rUs&NXD3+n;jB86`X?8(l3MBo6)PdakI6V6a}22{)8ilT zM~T*mU}__xSy|6XSrJ^%lDAR3Lft%+yxC|ZUvSO_nqMX!_ul3;R#*{~4DA=h$bP)%8Yv9X zyp><|e8=_ttI}ZAwOd#dlnSjck#6%273{E$kJuCGu=I@O)&6ID{nWF5@gLb16sj|&Sb~+du4e4O_%_o`Ix4NRrAsyr1_}MuP94s>de8cH-OUkVPk3+K z&jW)It9QiU-ti~AuJkL`XMca8Oh4$SyJ=`-5WU<{cIh+XVH#e4d&zive_UHC!pN>W z3TB;Mn5i)9Qn)#6@lo4QpI3jFYc0~+jS)4AFz8fVC;lD^+idw^S~Qhq>Tg(!3$yLD zzktzoFrU@6s4wwCMz}edpF5i5Q1IMmEJQHzp(LAt)pgN3&O!&d?3W@6U4)I^2V{;- z6A(?zd93hS*uQmnh4T)nHnE{wVhh(=MMD(h(P4+^p83Om6t<*cUW>l(qJzr%5vp@K zN27ka(L{JX=1~e2^)F^i=TYj&;<7jyUUR2Bek^A8+3Up*&Xwc{)1nRR5CT8vG>ExV zHnF3UqXJOAno_?bnhCX-&kwI~Ti8t4`n0%Up>!U`ZvK^w2+0Cs-b9%w%4`$+To|k= zKtgc&l}P`*8IS>8DOe?EB84^kx4BQp3<7P{Pq}&p%xF_81pg!l2|u=&I{AuUgmF5n zJQCTLv}%}xbFGYtKfbba{CBo)lWW%Z>i(_NvLhoQZ*5-@2l&x>e+I~0Nld3UI9tdL zRzu8}i;X!h8LHVvN?C+|M81e>Jr38%&*9LYQec9Ax>?NN+9(_>XSRv&6hlCYB`>Qm z1&ygi{Y()OU4@D_jd_-7vDILR{>o|7-k)Sjdxkjgvi{@S>6GqiF|o`*Otr;P)kLHN zZkpts;0zw_6;?f(@4S1FN=m!4^mv~W+lJA`&7RH%2$)49z0A+8@0BCHtj|yH--AEL z0tW6G%X-+J+5a{5*WKaM0QDznf;V?L5&uQw+yegDNDP`hA;0XPYc6e0;Xv6|i|^F2WB)Z$LR|HR4 zTQsRAby9(^Z@yATyOgcfQw7cKyr^3Tz7lc7+JEwwzA7)|2x+PtEb>nD(tpxJQm)Kn zW9K_*r!L%~N*vS8<5T=iv|o!zTe9k_2jC_j*7ik^M_ zaf%k{WX{-;0*`t`G!&`eW;gChVXnJ-Rn)To8vW-?>>a%QU1v`ZC=U)f8iA@%JG0mZ zDqH;~mgBnrCP~1II<=V9;EBL)J+xzCoiRBaeH&J6rL!{4zIY8tZka?_FBeQeNO3q6 zyG_alW54Ba&wQf{&F1v-r1R6ID)PTsqjIBc+5MHkcW5Fnvi~{-FjKe)t1bl}Y;z@< z=!%zvpRua>>t_x}^}z0<7MI!H2v6|XAyR9!t50q-A)xk0nflgF4*OQlCGK==4S|wc zRMsSscNhRzHMBU8TdcHN!q^I}x0iXJ%uehac|Zs_B$p@CnF)HeXPpB_Za}F{<@6-4 zl%kml@}kHQ(ypD8FsPJ2=14xXJE|b20RUIgs!2|R3>LUMGF6X*B_I|$`Qg=;zm7C z{mEDy9dTmPbued7mlO@phdmAmJ7p@GR1bjCkMw6*G7#4+`k>fk1czdJUB!e@Q(~6# zwo%@p@V5RL0ABU2LH7Asq^quDUho@H>eTZH9f*no9fY0T zD_-9px3e}A!>>kv5wk91%C9R1J_Nh!*&Kk$J3KNxC}c_@zlgpJZ+5L)Nw|^p=2ue}CJtm;uj*Iqr)K})kA$xtNUEvX;4!Px*^&9T_`IN{D z{6~QY=Nau6EzpvufB^hflc#XIsSq0Y9(nf$d~6ZwK}fal92)fr%T3=q{0mP-EyP_G z)UR5h@IX}3Qll2b0oCAcBF>b*@Etu*aTLPU<%C>KoOrk=x?pN!#f_Og-w+;xbFgjQ zXp`et%lDBBh~OcFnMKMUoox0YwBNy`N0q~bSPh@+enQ=4RUw1) zpovN`QoV>vZ#5LvC;cl|6jPr}O5tu!Ipoyib8iXqy}TeJ;4+_7r<1kV0v5?Kv>fYp zg>9L`;XwXa&W7-jf|9~uP2iyF5`5AJ`Q~p4eBU$MCC00`rcSF>`&0fbd^_eqR+}mK z4n*PMMa&FOcc)vTUR zlDUAn-mh`ahi_`f`=39JYTNVjsTa_Y3b1GOIi)6dY)D}xeshB0T8Eov5%UhWd1)u}kjEQ|LDo{tqKKrYIfVz~@dp!! zMOnah@vp)%_-jDTUG09l+;{CkDCH|Q{NqX*uHa1YxFShy*1+;J`gywKaz|2Q{lG8x zP?KBur`}r`!WLKXY_K;C8$EWG>jY3UIh{+BLv0=2)KH%P}6xE2kg)%(-uA6lC?u8}{K(#P*c zE9C8t*u%j2r_{;Rpe1A{9nNXU;b_N0vNgyK!EZVut~}+R2rcbsHilqsOviYh-pYX= zHw@53nlmwYI5W5KP>&`dBZe0Jn?nAdC^HY1wlR6$u^PbpB#AS&5L6zqrXN&7*N2Q` z+Rae1EwS)H=aVSIkr8Ek^1jy2iS2o7mqm~Mr&g5=jjt7VxwglQ^`h#Mx+x2v|9ZAwE$i_9918MjJxTMr?n!bZ6n$}y11u8I9COTU`Z$Fi z!AeAQLMw^gp_{+0QTEJrhL424pVDp%wpku~XRlD3iv{vQ!lAf!_jyqd_h}+Tr1XG| z`*FT*NbPqvHCUsYAkFnM`@l4u_QH&bszpUK#M~XLJt{%?00GXY?u_{gj3Hvs!=N(I z(=AuWPijyoU!r?aFTsa8pLB&cx}$*%;K$e*XqF{~*rA-qn)h^!(-;e}O#B$|S~c+U zN4vyOK0vmtx$5K!?g*+J@G1NmlEI=pyZXZ69tAv=@`t%ag_Hk{LP~OH9iE)I= zaJ69b4kuCkV0V zo(M0#>phpQ_)@j;h%m{-a*LGi(72TP)ws2w*@4|C-3+;=5DmC4s7Lp95%n%@Ko zfdr3-a7m*dys9iIci$A=4NPJ`HfJ;hujLgU)ZRuJI`n;Pw|yksu!#LQnJ#dJysgNb z@@qwR^wrk(jbq4H?d!lNyy72~Dnn87KxsgQ!)|*m(DRM+eC$wh7KnS-mho3|KE)7h zK3k;qZ;K1Lj6uEXLYUYi)1FN}F@-xJ z@@3Hb84sl|j{4$3J}aTY@cbX@pzB_qM~APljrjju6P0tY{C@ zpUCOz_NFmALMv1*blCcwUD3?U6tYs+N%cmJ98D%3)%)Xu^uvzF zS5O!sc#X6?EwsYkvPo6A%O8&y8sCCQH<%f2togVwW&{M;PR!a(ZT_A+jVAbf{@5kL zB@Z(hb$3U{T_}SKA_CoQVU-;j>2J=L#lZ~aQCFg-d<9rzs$_gO&d5N6eFSc z1ml8)P*FSi+k@!^M9nDWR5e@ATD8oxtDu=36Iv2!;dZzidIS(PCtEuXAtlBb1;H%Z zwnC^Ek*D)EX4#Q>R$$WA2sxC_t(!!6Tr?C#@{3}n{<^o;9id1RA&-Pig1e-2B1XpG zliNjgmd3c&%A}s>qf{_j#!Z`fu0xIwm4L0)OF=u(OEmp;bLCIaZX$&J_^Z%4Sq4GZ zPn6sV_#+6pJmDN_lx@1;Zw6Md_p0w9h6mHtzpuIEwNn>OnuRSC2=>fP^Hqgc)xu^4 z<3!s`cORHJh#?!nKI`Et7{3C27+EuH)Gw1f)aoP|B3y?fuVfvpYYmmukx0ya-)TQX zR{ggy5cNf4X|g)nl#jC9p>7|09_S7>1D2GTRBUTW zAkQ=JMRogZqG#v;^=11O6@rPPwvJkr{bW-Qg8`q8GoD#K`&Y+S#%&B>SGRL>;ZunM@49!}Uy zN|bBCJ%sO;@3wl0>0gbl3L@1^O60ONObz8ZI7nder>(udj-jt`;yj^nTQ$L9`OU9W zX4alF#$|GiR47%x@s&LV>2Sz2R6?;2R~5k6V>)nz!o_*1Y!$p>BC5&?hJg_MiE6UBy>RkVZj`9UWbRkN-Hk!S`=BS3t3uyX6)7SF#)71*}`~Ogz z1rap5H6~dhBJ83;q-Y<5V35C2&F^JI-it(=5D#v!fAi9p#UwV~2tZQI+W(Dv?1t9? zfh*xpxxO{-(VGB>!Q&0%^YW_F!@aZS#ucP|YaD#>wd1Fv&Z*SR&mc;asi}1G) z_H>`!akh-Zxq9#io(7%;a$)w+{QH)Y$?UK1Dt^4)up!Szcxnu}kn$0afcfJL#IL+S z5gF_Y30j;{lNrG6m~$Ay?)*V9fZuU@3=kd40=LhazjFrau>(Y>SJNtOz>8x_X-BlA zIpl{i>OarVGj1v(4?^1`R}aQB&WCRQzS~;7R{tDZG=HhgrW@B`W|#cdyj%YBky)P= zpxuOZkW>S6%q7U{VsB#G(^FMsH5QuGXhb(sY+!-R8Bmv6Sx3WzSW<1MPPN1!&PurYky(@`bP9tz z52}LH9Q?+FF5jR6-;|+GVdRA!qtd;}*-h&iIw3Tq3qF9sDIb1FFxGbo&fbG5n8$3F zyY&PWL{ys^dTO}oZ#@sIX^BKW*bon=;te9j5k+T%wJ zNJtoN1~YVj4~YRrlZl)b&kJqp+Z`DqT!la$x&&IxgOQw#yZd-nBP3!7FijBXD|IsU8Zl^ zc6?MKpJQ+7ka|tZQLfchD$PD|;K(9FiLE|eUZX#EZxhG!S-63C$jWX1Yd!6-Yxi-u zjULIr|0-Q%D9jz}IF~S%>0(jOqZ(Ln<$9PxiySr&2Oic7vb<8q=46)Ln%Z|<*z5&> z3f~Zw@m;vR(bESB<=Jqkxn(=#hQw42l(7)h`vMQQTttz9XW6^|^8EK7qhju4r_c*b zJIi`)MB$w@9epwdIfnEBR+?~);yd6C(LeMC& zn&&N*?-g&BBJcV;8&UoZi4Lmxcj16ojlxR~zMrf=O_^i1wGb9X-0@6_rpjPYemIin zmJb+;lHe;Yp=8G)Q(L1bzH*}I>}uAqhj4;g)PlvD9_e_ScR{Ipq|$8NvAvLD8MYr}xl=bU~)f%B3E>r3Bu9_t|ThF3C5~BdOve zEbk^r&r#PT&?^V1cb{72yEWH}TXEE}w>t!cY~rA+hNOTK8FAtIEoszp!qqptS&;r$ zaYV-NX96-h$6aR@1xz6_E0^N49mU)-v#bwtGJm)ibygzJ8!7|WIrcb`$XH~^!a#s& z{Db-0IOTFq#9!^j!n_F}#Z_nX{YzBK8XLPVmc&X`fT7!@$U-@2KM9soGbmOSAmqV z{nr$L^MBo_u^Joyf0E^=eo{Rt0{{e$IFA(#*kP@SQd6lWT2-#>` zP1)7_@IO!9lk>Zt?#CU?cuhiLF&)+XEM9B)cS(gvQT!X3`wL*{fArTS;Ak`J<84du zALKPz4}3nlG8Fo^MH0L|oK2-4xIY!~Oux~1sw!+It)&D3p;+N8AgqKI`ld6v71wy8I!eP0o~=RVcFQR2Gr(eP_JbSytoQ$Yt}l*4r@A8Me94y z8cTDWhqlq^qoAhbOzGBXv^Wa4vUz$(7B!mX`T=x_ueKRRDfg&Uc-e1+z4x$jyW_Pm zp?U;-R#xt^Z8Ev~`m`iL4*c#65Nn)q#=Y0l1AuD&+{|8-Gsij3LUZXpM0Bx0u7WWm zH|%yE@-#XEph2}-$-thl+S;__ciBxSSzHveP%~v}5I%u!z_l_KoW{KRx2=eB33umE zIYFtu^5=wGU`Jab8#}cnYry@9p5UE#U|VVvx_4l49JQ;jQdp(uw=$^A$EA$LM%vmE zvdEOaIcp5qX8wX{mYf0;#51~imYYPn4=k&#DsKTxo{_Mg*;S495?OBY?#gv=edYC* z^O@-sd-qa+U24xvcbL0@C7_6o!$`)sVr-jSJE4XQUQ$?L7}2(}Eixqv;L8AdJAVqc zq}RPgpnDb@E_;?6K58r3h4-!4rT4Ab#rLHLX?eMOfluJk=3i1@Gt1i#iA=O`M0@x! z(HtJP9BMHXEzuD93m|B&woj0g6T?f#^)>J>|I4C5?Gam>n9!8CT%~aT;=oco5d6U8 zMXl(=W;$ND_8+DD*?|5bJ!;8ebESXMUKBAf7YBwNVJibGaJ*(2G`F%wx)grqVPjudiaq^Kl&g$8A2 zWMxMr@_$c}d+;_B`#kUX-t|4VKH&_f^^EP0&=DPLW)H)UzBG%%Tra*5 z%$kyZe3I&S#gfie^z5)!twG={3Cuh)FdeA!Kj<-9** zvT*5%Tb`|QbE!iW-XcOuy39>D3oe6x{>&<#E$o8Ac|j)wq#kQzz|ATd=Z0K!p2$QE zPu?jL8Lb^y3_CQE{*}sTDe!2!dtlFjq&YLY@2#4>XS`}v#PLrpvc4*@q^O{mmnr5D zmyJq~t?8>FWU5vZdE(%4cuZuao0GNjp3~Dt*SLaxI#g_u>hu@k&9Ho*#CZP~lFJHj z(e!SYlLigyc?&5-YxlE{uuk$9b&l6d`uIlpg_z15dPo*iU&|Khx2*A5Fp;8iK_bdP z?T6|^7@lcx2j0T@x>X7|kuuBSB7<^zeY~R~4McconTxA2flHC0_jFxmSTv-~?zVT| zG_|yDqa9lkF*B6_{j=T>=M8r<0s;@z#h)3BQ4NLl@`Xr__o7;~M&dL3J8fP&zLfDfy z);ckcTev{@OUlZ`bCo(-3? z1u1xD`PKgSg?RqeVVsF<1SLF;XYA@Bsa&cY!I48ZJn1V<3d!?s=St?TLo zC0cNr`qD*M#s6f~X>SCNVkva^9A2ZP>CoJ9bvgXe_c}WdX-)pHM5m7O zrHt#g$F0AO+nGA;7dSJ?)|Mo~cf{z2L)Rz!`fpi73Zv)H=a5K)*$5sf_IZypi($P5 zsPwUc4~P-J1@^3C6-r9{V-u0Z&Sl7vNfmuMY4yy*cL>_)BmQF!8Om9Dej%cHxbIzA zhtV0d{=%cr?;bpBPjt@4w=#<>k5ee=TiWAXM2~tUGfm z$s&!Dm0R^V$}fOR*B^kGaipi~rx~A2cS0;t&khV1a4u38*XRUP~f za!rZMtay8bsLt6yFYl@>-y^31(*P!L^^s@mslZy(SMsv9bVoX`O#yBgEcjCmGpyc* zeH$Dw6vB5P*;jor+JOX@;6K#+xc)Z9B8M=x2a@Wx-{snPGpRmOC$zpsqW*JCh@M2Y z#K+M(>=#d^>Of9C`))h<=Bsy)6zaMJ&x-t%&+UcpLjV`jo4R2025 zXaG8EA!0lQa)|dx-@{O)qP6`$rhCkoQqZ`^SW8g-kOwrwsK8 z3ms*AIcyj}-1x&A&vSq{r=QMyp3CHdWH35!sad#!Sm>^|-|afB+Q;|Iq@LFgqIp#Z zD1%H+3I?6RGnk&IFo|u+E0dCxXz4yI^1i!QTu7uvIEH>i3rR{srcST`LIRwdV1P;W z+%AN1NIf@xxvVLiSX`8ILA8MzNqE&7>%jMzGt9wm78bo9<;h*W84i29^w!>V>{N+S zd`5Zmz^G;f=icvoOZfK5#1ctx*~UwD=ab4DGQXehQ!XYnak*dee%YN$_ZPL%KZuz$ zD;$PpT;HM^$KwtQm@7uvT`i6>Hae1CoRVM2)NL<2-k2PiX=eAx+-6j#JI?M}(tuBW zkF%jjLR)O`gI2fcPBxF^HeI|DWwQWHVR!;;{BXXHskxh8F@BMDn`oEi-NHt;CLymW z=KSv5)3dyzec0T5B*`g-MQ<;gz=nIWKUi9ko<|4I(-E0k$QncH>E4l z**1w&#={&zv4Tvhgz#c29`m|;lU-jmaXFMC11 z*dlXDMEOG>VoLMc>!rApwOu2prKSi*!w%`yzGmS+k(zm*CsLK*wv{S_0WX^8A-rKy zbk^Gf_92^7iB_uUF)EE+ET4d|X|>d&mdN?x@vxKAQk`O+r4Qdu>XGy(a(19g;=jU} zFX{O*_NG>!$@jh!U369Lnc+D~qch3uT+_Amyi}*k#LAAwh}k8IPK5a-WZ81ufD>l> z$4cF}GSz>ce`3FAic}6W4Z7m9KGO?(eWqi@L|5Hq0@L|&2flN1PVl}XgQ2q*_n2s3 zt5KtowNkTYB5b;SVuoXA@i5irXO)A&%7?V`1@HGCB&)Wgk+l|^XXChq;u(nyPB}b3 zY>m5jkxpZgi)zfbgv&ec4Zqdvm+D<?Im*mXweS9H+V>)zF#Zp3)bhl$PbISY{5=_z!8&*Jv~NYtI-g!>fDs zmvL5O^U%!^VaKA9gvKw|5?-jk>~%CVGvctKmP$kpnpfN{D8@X*Aazi$txfa%vd-|E z>kYmV66W!lNekJPom29LdZ%(I+ZLZYTXzTg*to~m?7vp%{V<~>H+2}PQ?PPAq`36R z<%wR8v6UkS>Wt#hzGk#44W<%9S=nBfB);6clKwnxY}T*w21Qc3_?IJ@4gYzC7s;WP zVQNI(M=S=JT#xsZy7G`cR(BP9*je0bfeN8JN5~zY(DDs0t{LpHOIbN);?T-69Pf3R zSNe*&p2%AwXHL>__g+xd4Hlc_vu<25H?(`nafS%)3UPP7_4;gk-9ckt8SJRTv5v0M z_Hww`qPudL?ajIR&X*;$y-`<)6dxx1U~5eGS13CB!lX;3w7n&lDDiArbAhSycd}+b zya_3p@A`$kQy;|NJZ~s44Hqo7Hwt}X86NK=(ey>lgWTtGL6k@Gy;PbO!M%1~Wcn2k zUFP|*5d>t-X*RU8g%>|(wwj*~#l4z^Aatf^DWd1Wj#Q*AY0D^V@sC`M zjJc6qXu0I7Y*2;;gGu!plAFzG=J;1%eIOdn zQA>J&e05UN*7I5@yRhK|lbBSfJ+5Uq;!&HV@xfPZrgD}kE*1DSq^=%{o%|LChhl#0 zlMb<^a6ixzpd{kNZr|3jTGeEzuo}-eLT-)Q$#b{!vKx8Tg}swCni>{#%vDY$Ww$84 zew3c9BBovqb}_&BRo#^!G(1Eg((BScRZ}C)Oz?y`T5wOrv);)b^4XR8 zhJo7+<^7)qB>I;46!GySzdneZ>n_E1oWZY;kf94#)s)kWjuJN1c+wbVoNQcmnv}{> zN0pF+Sl3E}UQ$}slSZeLJrwT>Sr}#V(dVaezCQl2|4LN`7L7v&siYR|r7M(*JYfR$ zst3=YaDw$FSc{g}KHO&QiKxuhEzF{f%RJLKe3p*7=oo`WNP)M(9X1zIQPP0XHhY3c znrP{$4#Ol$A0s|4S7Gx2L23dv*Gv2o;h((XVn+9+$qvm}s%zi6nI-_s6?mG! zj{DV;qesJb&owKeEK?=J>UcAlYckA7Sl+I&IN=yasrZOkejir*kE@SN`fk<8Fgx*$ zy&fE6?}G)d_N`){P~U@1jRVA|2*69)KSe_}!~?+`Yb{Y=O~_+@!j<&oVQQMnhoIRU zA0CyF1OFfkK44n*JD~!2!SCPM;PRSk%1XL=0&rz00wxPs&-_eapJy#$h!eqY%nS0{ z!aGg58JIJPF3_ci%n)QSVpa2H`vIe$RD43;#IRfDV&Ibit z+?>HW4{2wOfC6Fw)}4x}i1maDxcE1qi@BS*qcxD2gE@h3#4cgU*D-&3z7D|tVZWt= z-Cy2+*Cm@P4GN_TPUtaVyVesbVDazF@)j8VJ4>XZv!f%}&eO1SvIgr}4`A*3#vat< z_MoByL(qW6L7SFZ#|Gc1fFN)L2PxY+{B8tJp+pxRyz*87)vXR}*=&ahXjBlQKguuf zX6x<<6fQulE^C*KH8~W%ptpaC0l?b=_{~*U4?5Vt;dgM4t_{&UZ1C2j?b>b+5}{IF_CUyvz-@QZPMlJ)r_tS$9kH%RPv#2_nMb zRLj5;chJ72*U`Z@Dqt4$@_+k$%|8m(HqLG!qT4P^DdfvGf&){gKnGCX#H0!;W=AGP zbA&Z`-__a)VTS}kKFjWGk z%|>yE?t*EJ!qeQ%dPk$;xIQ+P0;()PCBDgjJm6Buj{f^awNoVx+9<|lg3%-$G(*f) zll6oOkN|yamn1uyl2*N-lnqRI1cvs_JxLTeahEK=THV$Sz*gQhKNb*p0fNoda#-&F zB-qJgW^g}!TtM|0bS2QZekW7_tKu%GcJ!4?lObt0z_$mZ4rbQ0o=^curCs3bJK6sq z9fu-aW-l#>z~ca(B;4yv;2RZ?tGYAU)^)Kz{L|4oPj zdOf_?de|#yS)p2v8-N||+XL=O*%3+y)oI(HbM)Ds?q8~HPzIP(vs*G`iddbWq}! z(2!VjP&{Z1w+%eUq^ /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +131,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +198,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/gradlew.bat b/gradlew.bat index 53a6b238..7101f8e4 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -42,11 +43,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail From 8b2839d371e462b693a574577a34acda1af225ba Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 12 Apr 2024 13:45:27 +0200 Subject: [PATCH 24/55] improve volume loading from scene --- .../iview/commands/demo/advanced/EyeTrackingDemo.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 499026c8..17453c4c 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -40,6 +40,7 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import graphics.scenery.volumes.RAIVolume @Plugin(type = Command::class, menuRoot = "SciView", @@ -90,6 +91,7 @@ class EyeTrackingDemo: Command{ override fun run() { sciview.toggleVRRendering() + log.info("VR mode has been toggled") hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) @@ -121,7 +123,10 @@ class EyeTrackingDemo: Command{ shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) sciview.addChild(shell) - volume = sciview.find("volume") as Volume +// volume = sciview.find("volume") as Volume + val volnodes = sciview.findNodes { node -> node.nodeType.equals(RAIVolume::class.java) } + log.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") + volume = volnodes.first() as Volume volume.visible = false val bb = BoundingGrid() @@ -191,6 +196,7 @@ class EyeTrackingDemo: Command{ } } thread{ + log.info("started thread for inputSetup") inputSetup() } thread { @@ -403,7 +409,7 @@ class EyeTrackingDemo: Command{ hmd.addKeyBinding("cell_division", "T") hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - + log.info("calibration should start now") setupCalibration() } @@ -415,6 +421,7 @@ class EyeTrackingDemo: Command{ val cam = sciview.camera as? DetachedHeadCamera ?: return@thread pupilTracker.gazeConfidenceThreshold = confidenceThreshold if (!pupilTracker.isCalibrated) { + log.info("pupil is currently uncalibrated") pupilTracker.onCalibrationInProgress = { cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) } From 6136a3c7ed3db2eaa082ed1a011e3256a690761f Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 12 Apr 2024 15:34:21 +0200 Subject: [PATCH 25/55] fix filtering for volumes in the scene --- .../kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 17453c4c..d983dac2 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -41,6 +41,7 @@ import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import graphics.scenery.volumes.RAIVolume +import kotlin.reflect.KClass @Plugin(type = Command::class, menuRoot = "SciView", @@ -124,7 +125,7 @@ class EyeTrackingDemo: Command{ sciview.addChild(shell) // volume = sciview.find("volume") as Volume - val volnodes = sciview.findNodes { node -> node.nodeType.equals(RAIVolume::class.java) } + val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } log.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") volume = volnodes.first() as Volume volume.visible = false From a754d0964dab1d20bb28cc0c21fdf194e28d6d66 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 7 May 2024 17:04:24 +0200 Subject: [PATCH 26/55] SciView: fix VR toggling changes to stereo rendering --- src/main/kotlin/sc/iview/SciView.kt | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 16459a9b..958413c5 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -1795,6 +1795,8 @@ class SciView : SceneryBase, CalibratedRealInterval { * Enable VR rendering */ fun toggleVRRendering() { + var renderer = renderer ?: return + vrActive = !vrActive val cam = scene.activeObserver as? DetachedHeadCamera ?: return var ti: TrackerInput? = null @@ -1817,25 +1819,29 @@ class SciView : SceneryBase, CalibratedRealInterval { } if (vrActive && ti != null) { cam.tracker = ti + logger.info("tracker set") } else { cam.tracker = null } - renderer!!.pushMode = false + renderer.pushMode = false + // we need to force reloading the renderer as the HMD might require device or instance extensions if (renderer is VulkanRenderer && hmdAdded) { - replaceRenderer((renderer as VulkanRenderer).javaClass.simpleName, true, true) - (renderer as VulkanRenderer).toggleVR() - while (!(renderer as VulkanRenderer).initialized /* || !getRenderer().getFirstImageReady()*/) { - logger.debug("Waiting for renderer reinitialisation") + replaceRenderer(renderer.javaClass.simpleName, true, true) + + logger.info("renderer replaced") + while (renderer.initialized == false || renderer.firstImageReady == false) { + renderer = this.renderer!! + logger.info("Waiting for renderer reinitialisation (init: ${renderer.initialized} ready: ${renderer.firstImageReady}") try { Thread.sleep(200) } catch (e: InterruptedException) { e.printStackTrace() } } - } else { - renderer!!.toggleVR() } + + renderer.toggleVR() } /** From bbca486941e37ed1f9ffdede08b3bcdd09516d27 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 7 May 2024 17:05:19 +0200 Subject: [PATCH 27/55] EyeTrackingDemo: catch if no volume is found --- .../commands/demo/advanced/EyeTrackingDemo.kt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index d983dac2..6e0b6842 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -126,8 +126,15 @@ class EyeTrackingDemo: Command{ // volume = sciview.find("volume") as Volume val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } - log.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") - volume = volnodes.first() as Volume + + val v = (volnodes.firstOrNull() as? Volume) + if(v == null) { + log.warn("No volume found, bailing") + return + } else { + log.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") + volume = v + } volume.visible = false val bb = BoundingGrid() @@ -165,7 +172,8 @@ class EyeTrackingDemo: Command{ val image = ImageIO.read(stream) val data = (image.raster.dataBuffer as DataBufferByte).data - node.ifMaterial {textures["diffuse"] = Texture( + node.ifMaterial { + textures["diffuse"] = Texture( Vector3i(image.width, image.height, 1), 3, UnsignedByteType(), @@ -175,6 +183,7 @@ class EyeTrackingDemo: Command{ lastFrame = System.nanoTime() } + // TODO: Replace with cam.showMessage() val debugBoard = TextBoard() debugBoard.name = "debugBoard" debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) @@ -529,7 +538,8 @@ class EyeTrackingDemo: Command{ // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + // TODO: Allow for sampling a given time point of a volume + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) if (samples != null && localDirection != null) { val metadata = SpineMetadata( @@ -550,6 +560,7 @@ class EyeTrackingDemo: Command{ spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + // TODO: Show confidence as color for the spine spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } } } From 6fe080597be23952b760b4e112a253ce83de2b05 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 7 May 2024 17:05:46 +0200 Subject: [PATCH 28/55] HedgehogAnalysis: add docs and comments --- .../demo/advanced/HedgehogAnalysis.kt | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 471b7684..8e83940d 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -50,6 +50,10 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix avgConfidence /= totalSampleCount } + /** + * From a [list] of Floats, return both the index of local maxima, and their value, + * packaged nicely as a Pair + */ private fun localMaxima(list: List): List> = list.windowed(3, 1).mapIndexed { index, l -> val left = l[0] @@ -104,6 +108,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix return Quaternionf(q.x/(2.0f * x), q.y/(2.0f * x), q.z/(2.0f * x), x) } + data class VertexWithDistance(val vertex: SpineGraphVertex, val distance: Float) + fun run(): Track? { val startingThreshold = 0.002f @@ -123,6 +129,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix logger.info("Starting point is ${startingPoint.key}/${timepoints.size} (threshold=$startingThreshold)") + // filter timepoints, remove all before the starting point timepoints.filter { it.key > startingPoint.key } .forEach { timepoints.remove(it.key) } @@ -130,16 +137,22 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix //step2: find the maxIndices along the spine - val candidates = timepoints.map { tp -> + // yo dawg, this will be a list of lists, where each entry in the first-level list + // corresponds to a time point, which then contains a list of vertices within that timepoint. + val candidates: List> = timepoints.map { tp -> val vs = tp.value.mapIndexedNotNull { i, spine -> + // determine local maxima (and their indices) along the spine, aka, actual things the user might have + // seen when looking into the direction of the spine val maxIndices = localMaxima(spine.samples.filterNotNull()) -// logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata if(maxIndices.isNotEmpty()) { - maxIndices. + //maxIndices. // filter the maxIndices which are too far away, which can be removed - filter { it.first <1200}. - map { index -> + //filter { it.first <1200}. + maxIndices.map { index -> + logger.info("Generating vertex at index $index") val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() SpineGraphVertex(tp.key, @@ -157,33 +170,38 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix vs }.flatten() + logger.info("SpineGraphVertices extracted") - //step3: connect localMaximal points between 2 candidate spines according to the shortest path principle - // get the initial vertex, this one is assumed to always be in front, and have a local max - val initial = candidates.first().filter{it.value>startingThreshold}.first() + // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle + // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what + // the user looks at first is assumed to be the actual cell they want to track + val initial = candidates.first().filter { it.value>startingThreshold }.first() var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> - val distances = vs + // calculate world-space distances between current point, and all candidate + // vertices, sorting them by distance + val vertices = vs .filter { it.value > localMaxThreshold } .map { vertex -> val t = current.worldPosition - vertex.worldPosition val distance = t.length() - vertex to distance + VertexWithDistance(vertex, distance) } - .sortedBy { it.second } - - val closest = distances.firstOrNull()?.first - if(closest != null && distances.firstOrNull()?.second!! >0) { - current.next = closest - closest.previous = current - current = closest + .sortedBy { it.distance } + + val closest = vertices.firstOrNull() + if(closest != null && closest.distance > 0) { + // create a linked list between current and closest vertices + current.next = closest.vertex + closest.vertex.previous = current + current = closest.vertex current } else { null } }.toMutableList() - + // calculate average path lengths over all val beforeCount = shortestPath.size var avgPathLength = shortestPath.map { it.distance() }.average().toFloat() var stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() @@ -195,6 +213,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix while (shortestPath.any { it.distance() >= removeTooFarThreshold * avgPathLength }) { shortestPath = shortestPath.filter { it.distance() < removeTooFarThreshold * avgPathLength }.toMutableList() shortestPath.windowed(3, 1, partialWindows = true).forEach { + // this reconnects the neighbors after the offending vertex has been removed it.getOrNull(0)?.next = it.getOrNull(1) it.getOrNull(1)?.previous = it.getOrNull(0) it.getOrNull(1)?.next = it.getOrNull(2) @@ -203,10 +222,11 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } + // recalculate statistics after offending vertex removal avgPathLength = shortestPath.map { it.distance() }.average().toFloat() stdDevPathLength = shortestPath.map { it.distance() }.stddev().toFloat() - //step5: remove some edges according to zscoreThreshold + //step5: remove some vertices according to zscoreThreshold var remaining = shortestPath.count { zScore(it.distance(), avgPathLength, stdDevPathLength) > zscoreThreshold } logger.info("Iterating: ${shortestPath.size} vertices remaining, with $remaining failing z-score criterion") while(remaining > 0) { From da9979a512e80ec4f9bacdeb746b8593f334bac0 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:54:48 +0200 Subject: [PATCH 29/55] Add: test object for loading a volume and starting the eye tracking directly (using ugly hardcoded path) --- .../sc/iview/StartEyeTrackingDirectly.kt | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt new file mode 100644 index 00000000..77767cfb --- /dev/null +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -0,0 +1,38 @@ +package sc.iview + +import graphics.scenery.utils.lazyLogger +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.TransferFunction +import org.scijava.command.CommandService +import org.scijava.ui.UIService +import sc.iview.commands.demo.advanced.EyeTrackingDemo + +object StartEyeTrackingDirectly { + + val logger by lazyLogger() + + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val context = sv.scijavaContext + val uiService = context?.service(UIService::class.java) + uiService?.showUI() + + sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") + val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } + volumes.first().let { + it as RAIVolume + it.minDisplayRange = 400f + it.maxDisplayRange = 1500f + val tf = TransferFunction() + tf.addControlPoint(0f, 0f) + tf.addControlPoint(1f, 1f) + it.transferFunction = tf + } + + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) + + } +} \ No newline at end of file From 8c6c93ab0f2da7e96b41f1d54b28f79d47c5ecd4 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:03:27 +0200 Subject: [PATCH 30/55] Make newest sciview compatible with local scenery (stupid jackson) --- build.gradle.kts | 88 ++++++++++--------- gradle.properties | 2 +- .../sc/iview/StartEyeTrackingDirectly.kt | 51 ++++++----- 3 files changed, 71 insertions(+), 70 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8418f8cd..6da63a56 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -149,29 +149,29 @@ dependencies { val isRelease: Boolean get() = System.getProperty("release") == "true" -kotlin { - jvmToolchain(21) -// compilerOptions { -// jvmTarget = JvmTarget.JVM_21 -// freeCompilerArgs = listOf("-Xinline-classes", "-opt-in=kotlin.RequiresOptIn") -// } -} - -java { - targetCompatibility = JavaVersion.VERSION_21 - sourceCompatibility = JavaVersion.VERSION_21 -} +//kotlin { +// jvmToolchain(21) +//// compilerOptions { +//// jvmTarget = JvmTarget.JVM_21 +//// freeCompilerArgs = listOf("-Xinline-classes", "-opt-in=kotlin.RequiresOptIn") +//// } +//} +// +//java { +// targetCompatibility = JavaVersion.VERSION_21 +// sourceCompatibility = JavaVersion.VERSION_21 +//} tasks { -// withType().all { -// val version = System.getProperty("java.version").substringBefore('.').toInt() -// val default = if (version == 1) "21" else "$version" -// kotlinOptions { -// jvmTarget = project.properties["jvmTarget"]?.toString() ?: default -// freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") -// } -//// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default -// } + withType().all { + val version = System.getProperty("java.version").substringBefore('.').toInt() + val default = if (version == 1) "21" else "$version" + kotlinOptions { + jvmTarget = project.properties["jvmTarget"]?.toString() ?: default + freeCompilerArgs += listOf("-Xinline-classes", "-Xopt-in=kotlin.RequiresOptIn") + } +// sourceCompatibility = project.properties["sourceCompatibility"]?.toString() ?: default + } test { finalizedBy(jacocoTestReport) // report is always generated after tests run } @@ -428,30 +428,32 @@ tasks { register(name = "run") { classpath = sourceSets.main.get().runtimeClasspath - if (project.hasProperty("target")) { - project.property("target")?.let { target -> - classpath = sourceSets.test.get().runtimeClasspath - - println("Target is $target") - // if(target.endsWith(".kt")) { - // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") - // } else { - // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") - // } - - mainClass.set("$target") - val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } + var target: Any? = null + if (project.hasProperty("target")) + target = project.property("target") + target = "StartEyeTrackingDirectlyKt" + if (target != null) { + classpath = sourceSets.test.get().runtimeClasspath + + println("Target is $target") + // if(target.endsWith(".kt")) { + // main = target.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") + // } else { + // main = target.substringAfter("java${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".java") + // } + + mainClass.set("$target") + val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - println("Will run target $target with classpath $classpath, main=${mainClass.get()}") - println("JVM arguments passed to target: $allJvmArgs") + val additionalArgs = System.getenv("SCENERY_JVM_ARGS") + allJvmArgs = if (additionalArgs != null) { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs + } else { + allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } } + + println("Will run target $target with classpath $classpath, main=${mainClass.get()}") + println("JVM arguments passed to target: $allJvmArgs") } } diff --git a/gradle.properties b/gradle.properties index 91aaef27..a380f1c7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=2g org.gradle.caching=true jvmTarget=21 -#useLocalScenery=true +useLocalScenery=true kotlinVersion=1.9.23 dokkaVersion=1.9.10 scijavaParentPOMVersion=37.0.0 diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt index 77767cfb..33164755 100644 --- a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -1,38 +1,37 @@ -package sc.iview - import graphics.scenery.utils.lazyLogger import graphics.scenery.volumes.RAIVolume import graphics.scenery.volumes.TransferFunction import org.scijava.command.CommandService import org.scijava.ui.UIService +import sc.iview.SciView import sc.iview.commands.demo.advanced.EyeTrackingDemo -object StartEyeTrackingDirectly { +//object StartEye { - val logger by lazyLogger() +// val logger by lazyLogger() - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val context = sv.scijavaContext - val uiService = context?.service(UIService::class.java) - uiService?.showUI() +// @JvmStatic +fun main() { + val sv = SciView.create() + val context = sv.scijavaContext + val uiService = context?.service(UIService::class.java) + uiService?.showUI() - sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") - val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } - volumes.first().let { - it as RAIVolume - it.minDisplayRange = 400f - it.maxDisplayRange = 1500f - val tf = TransferFunction() - tf.addControlPoint(0f, 0f) - tf.addControlPoint(1f, 1f) - it.transferFunction = tf - } + sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") + val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } + volumes.first().let { + it as RAIVolume + it.minDisplayRange = 400f + it.maxDisplayRange = 1500f + val tf = TransferFunction() + tf.addControlPoint(0f, 0f) + tf.addControlPoint(1f, 1f) + it.transferFunction = tf + } - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + command.run(EyeTrackingDemo::class.java, true, argmap) - } -} \ No newline at end of file +} +//} \ No newline at end of file From fc6dd31a1002f7795a4e3e3b4431b1dace907551 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:20:45 +0200 Subject: [PATCH 31/55] Fix toggleVRRendering --- src/main/kotlin/sc/iview/SciView.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/kotlin/sc/iview/SciView.kt b/src/main/kotlin/sc/iview/SciView.kt index 64f9f30f..8230a122 100644 --- a/src/main/kotlin/sc/iview/SciView.kt +++ b/src/main/kotlin/sc/iview/SciView.kt @@ -1867,8 +1867,6 @@ class SciView : SceneryBase, CalibratedRealInterval { renderer.toggleVR() } - - renderer.toggleVR() } /** From 4b4bd77a0c7b0616a573a15befcfaa93d380dc1d Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:39:59 +0200 Subject: [PATCH 32/55] Minor changes to fix finding local maxima, adding info logs --- .../commands/demo/advanced/EyeTrackingDemo.kt | 47 +++++++++---------- .../demo/advanced/HedgehogAnalysis.kt | 31 ++++++------ .../sc/iview/StartEyeTrackingDirectly.kt | 26 +++++----- 3 files changed, 53 insertions(+), 51 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 6e0b6842..794d12d0 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -41,6 +41,7 @@ import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import graphics.scenery.volumes.RAIVolume +import sc.iview.commands.demo.animation.ParticleDemo import kotlin.reflect.KClass @Plugin(type = Command::class, @@ -235,15 +236,17 @@ class EyeTrackingDemo: Command{ if(hedgehogs.visible) { if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + if (it.metadata.isNotEmpty()) { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } } } } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode hedgehog.instances.forEach { it.visible = true } } } @@ -265,10 +268,10 @@ class EyeTrackingDemo: Command{ } fun addHedgehog() { + log.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) + hedgehog.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) var hedgehogInstanced = InstancedNode(hedgehog) hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } @@ -364,11 +367,10 @@ class EyeTrackingDemo: Command{ }, confirmAction = { - hedgehogs.children.removeAt(hedgehogs.children.size-1) + hedgehogs.children.removeLast() volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> volume.removeChild(lastTrack) } - val hedgehogId = hedgehogIds.get() val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) @@ -456,10 +458,12 @@ class EyeTrackingDemo: Command{ val toggleTracking = ClickBehaviour { _, _ -> if (tracking) { + log.info("deactivating tracking...") referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) dumpHedgehog() } else { + log.info("activating tracking...") addHedgehog() referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) @@ -573,7 +577,8 @@ class EyeTrackingDemo: Command{ * If [hedgehog] is not null, the cell track will not be added to the scene. */ fun dumpHedgehog() { - var lastHedgehog = hedgehogs.children.last() as InstancedNode + log.info("dumping hedgehog...") + val lastHedgehog = hedgehogs.children.last() as InstancedNode val hedgehogId = hedgehogIds.incrementAndGet() val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() @@ -617,21 +622,11 @@ class EyeTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val master = if(lastHedgehog == null) { - val m = Cylinder(3f, 1.0f, 10) - m.ifMaterial { - ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - roughness = 1.0f - metallic = 0.0f - cullingMode = Material.CullingMode.None - } - m.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(m) - mInstanced - } else { - null - } + val m = Cylinder(3f, 1.0f, 10) + m.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + + m.name = "Track-$hedgehogId" + val master = InstancedNode(m) val parentId = 0 val volumeDimensions = volume.getDimensions() @@ -651,7 +646,7 @@ class EyeTrackingDemo: Command{ trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } - master?.let { volume.addChild(it) } + master.let { volume.addChild(it) } trackFileWriter.close() } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 8e83940d..2468af32 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -7,6 +7,7 @@ import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger import org.slf4j.LoggerFactory import java.io.File +import kotlin.math.log import kotlin.math.sqrt /** @@ -54,19 +55,20 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix * From a [list] of Floats, return both the index of local maxima, and their value, * packaged nicely as a Pair */ - private fun localMaxima(list: List): List> = - list.windowed(3, 1).mapIndexed { index, l -> - val left = l[0] - val center = l[1] - val right = l[2] - - // we have a match at center - if(left - center < 0 && center - right > 0) { - index + 1 to center - } else { - null - } - }.filterNotNull() + private fun localMaxima(list: List): List> { + return list.windowed(6, 2).mapIndexed { index, l -> + val left = l[0] + val center = l[2] + val right = l[4] + + // we have a match at center + if (left < center && center > right) { + index * 2 + 2 to center + } else { + null + } + }.filterNotNull() + } data class SpineGraphVertex(val timepoint: Int, val position: Vector3f, @@ -175,7 +177,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what // the user looks at first is assumed to be the actual cell they want to track - val initial = candidates.first().filter { it.value>startingThreshold }.first() + logger.info("candidates are: ${candidates.joinToString { ", " }}") + val initial = candidates.first().first { it.value > startingThreshold } var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> // calculate world-space distances between current point, and all candidate diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt index 33164755..ea0782f9 100644 --- a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -1,6 +1,8 @@ +import graphics.scenery.utils.extensions.times import graphics.scenery.utils.lazyLogger import graphics.scenery.volumes.RAIVolume import graphics.scenery.volumes.TransferFunction +import org.joml.Vector3f import org.scijava.command.CommandService import org.scijava.ui.UIService import sc.iview.SciView @@ -17,17 +19,19 @@ fun main() { val uiService = context?.service(UIService::class.java) uiService?.showUI() - sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") - val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } - volumes.first().let { - it as RAIVolume - it.minDisplayRange = 400f - it.maxDisplayRange = 1500f - val tf = TransferFunction() - tf.addControlPoint(0f, 0f) - tf.addControlPoint(1f, 1f) - it.transferFunction = tf - } + sv.open("C:/Software/datasets/MastodonTutorialDataset1/datasethdf5.xml") + val volumes = sv.findNodes { it.javaClass == RAIVolume::class.java } + volumes.first().let { + it as RAIVolume + it.minDisplayRange = 400f + it.maxDisplayRange = 1500f + val tf = TransferFunction() + tf.addControlPoint(0f, 0f) + tf.addControlPoint(1f, 1f) + it.transferFunction = tf + it.spatial().scale *= 20f + it.spatial().scale.z *= -1f + } val command = sv.scijavaContext!!.getService(CommandService::class.java) val argmap = HashMap() From 5e6f99cdd3f9848a2f4a506237e9bd42f8da7cd7 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:27:16 +0200 Subject: [PATCH 33/55] Increase volume size for testing dataset --- src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt index ea0782f9..88c812e5 100644 --- a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -29,7 +29,7 @@ fun main() { tf.addControlPoint(0f, 0f) tf.addControlPoint(1f, 1f) it.transferFunction = tf - it.spatial().scale *= 20f + it.spatial().scale *= 50f it.spatial().scale.z *= -1f } From f58cb0bf70a8539bdea84fc5f97f6fe141428bbe Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:07:03 +0200 Subject: [PATCH 34/55] Clean up logging --- .../sc/iview/commands/demo/advanced/HedgehogAnalysis.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 2468af32..645019f8 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -146,7 +146,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // determine local maxima (and their indices) along the spine, aka, actual things the user might have // seen when looking into the direction of the spine val maxIndices = localMaxima(spine.samples.filterNotNull()) - logger.info("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") + logger.debug("Local maxima at ${tp.key}/$i are: ${maxIndices.joinToString(",")}") // if there actually are local maxima, generate a graph vertex for them with all the necessary metadata if(maxIndices.isNotEmpty()) { @@ -154,7 +154,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // filter the maxIndices which are too far away, which can be removed //filter { it.first <1200}. maxIndices.map { index -> - logger.info("Generating vertex at index $index") + logger.debug("Generating vertex at index $index") val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() SpineGraphVertex(tp.key, @@ -177,7 +177,6 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix // step3: connect localMaximal points between 2 candidate spines according to the shortest path principle // get the initial vertex, this one is assumed to always be in front, and have a local maximum - aka, what // the user looks at first is assumed to be the actual cell they want to track - logger.info("candidates are: ${candidates.joinToString { ", " }}") val initial = candidates.first().first { it.value > startingThreshold } var current = initial var shortestPath = candidates.drop(1).mapIndexedNotNull { time, vs -> From e1fe8a3101337e4b2f7e9875e769fcb1951b89ac Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:07:48 +0200 Subject: [PATCH 35/55] Change showMessage to centered, try to fix track instancing --- .../commands/demo/advanced/EyeTrackingDemo.kt | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 794d12d0..2a94da68 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -6,7 +6,6 @@ import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole import graphics.scenery.controls.behaviours.ControllerDrag import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.numerics.Random import graphics.scenery.textures.Texture import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers @@ -40,9 +39,7 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import graphics.scenery.volumes.RAIVolume import sc.iview.commands.demo.animation.ParticleDemo -import kotlin.reflect.KClass @Plugin(type = Command::class, menuRoot = "SciView", @@ -304,17 +301,17 @@ class EyeTrackingDemo: Command{ HedgehogVisibility.Hidden -> { hedgehogs.visible = false hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) + cam.showMessage("Hedgehogs hidden",distance = 2f, size = 0.2f, centered = true) } HedgehogVisibility.PerTimePoint -> { hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) + cam.showMessage("Hedgehogs shown per timepoint",distance = 2f, size = 0.2f, centered = true) } HedgehogVisibility.Visible -> { hedgehogs.visible = true - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) + cam.showMessage("Hedgehogs visible",distance = 2f, size = 0.2f, centered = true) } } } @@ -330,7 +327,7 @@ class EyeTrackingDemo: Command{ val fasterOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f, centered = true) } else { volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) @@ -340,7 +337,7 @@ class EyeTrackingDemo: Command{ val slowerOrScale = ClickBehaviour { _, _ -> if(playing) { volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 2f, size = 0.2f, centered = true) } else { volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) @@ -350,9 +347,9 @@ class EyeTrackingDemo: Command{ val playPause = ClickBehaviour { _, _ -> playing = !playing if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) + cam.showMessage("Playing",distance = 2f, size = 0.2f, centered = true) } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) + cam.showMessage("Paused",distance = 2f, size = 0.2f, centered = true) } } @@ -360,10 +357,11 @@ class EyeTrackingDemo: Command{ val deleteLastHedgehog = ConfirmableClickBehaviour( armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) + cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt(), + centered = true) }, confirmAction = { @@ -379,10 +377,12 @@ class EyeTrackingDemo: Command{ hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") hedgehogFileWriter.close() - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) + cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000, + centered = true + ) }) hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> @@ -391,11 +391,11 @@ class EyeTrackingDemo: Command{ } else { PlaybackDirection.Forward } - cam.showMessage("Playing: ${direction}") + cam.showMessage("Playing: ${direction}", distance = 2f, centered = true) }) val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", duration = 1000) + cam.showMessage("Adding cell division", distance = 2f, duration = 1000) dumpHedgehog() addHedgehog() } @@ -435,15 +435,15 @@ class EyeTrackingDemo: Command{ if (!pupilTracker.isCalibrated) { log.info("pupil is currently uncalibrated") pupilTracker.onCalibrationInProgress = { - cam.showMessage("Crunching equations ...",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000) + cam.showMessage("Crunching equations ...",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000, centered = true) } pupilTracker.onCalibrationFailed = { - cam.showMessage("Calibration failed.",distance = 1.2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f)) + cam.showMessage("Calibration failed.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), centered = true) } pupilTracker.onCalibrationSuccess = { - cam.showMessage("Calibration succeeded!", distance = 1.2f, size = 0.2f,messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f)) + cam.showMessage("Calibration succeeded!", distance = 2f, size = 0.2f, messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), centered = true) // cam.children.find { it.name == "debugBoard" }?.visible = true for (i in 0 until 20) { @@ -460,13 +460,13 @@ class EyeTrackingDemo: Command{ if (tracking) { log.info("deactivating tracking...") referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) dumpHedgehog() } else { log.info("activating tracking...") addHedgehog() referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) } tracking = !tracking } @@ -482,7 +482,7 @@ class EyeTrackingDemo: Command{ sciview.deleteNode(sciview.find("eyeFrames")) log.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", distance = 1.2f, size = 0.15f,duration = 1500) + cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) pupilTracker.calibrate(cam, hmd, generateReferenceData = true, calibrationTarget = calibrationTarget) @@ -622,11 +622,16 @@ class EyeTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val m = Cylinder(3f, 1.0f, 10) - m.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) + val cylinder = Cylinder(0.01f, 1.0f, 6) + cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { + diffuse = Vector3f(1f) + ambient = Vector3f(1f) + roughness = 1f + } - m.name = "Track-$hedgehogId" - val master = InstancedNode(m) + cylinder.name = "Track-$hedgehogId" + val mainTrack = InstancedNode(cylinder) + mainTrack.instancedProperties["Color"] = { Vector4f(1f) } val parentId = 0 val volumeDimensions = volume.getDimensions() @@ -635,18 +640,19 @@ class EyeTrackingDemo: Command{ trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> - if(master != null) { - val element = master.addInstance() - element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) + if(mainTrack != null) { + val element = mainTrack.addInstance() + element.addAttribute(Material::class.java, cylinder.material()) + element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) element.parent = volume - master.instances.add(element) +// mainTrack.instances.add(element) } val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } - master.let { volume.addChild(it) } + mainTrack.let { sciview.addNode(it, parent = volume) } trackFileWriter.close() } From 8560296dd99cc5e32d7899b3258caaf005e7d43d Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 2 Sep 2024 16:58:20 +0200 Subject: [PATCH 36/55] Fix track cylinder radius --- .../kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 2a94da68..d5562772 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -622,7 +622,7 @@ class EyeTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val cylinder = Cylinder(0.01f, 1.0f, 6) + val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { diffuse = Vector3f(1f) ambient = Vector3f(1f) From 1d64c4ce110841996a4829ebccefcb2740947301 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:19:57 +0200 Subject: [PATCH 37/55] EyeTrackingDemo: update sciview API usage --- .../commands/demo/advanced/EyeTrackingDemo.kt | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index d5562772..6e0ba053 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -101,18 +101,19 @@ class EyeTrackingDemo: Command{ metallic = 0.0f diffuse = Vector3f(0.8f, 0.8f, 0.8f) } - sciview.camera!!.addChild(referenceTarget) + sciview.camera?.addChild(referenceTarget) calibrationTarget.visible = false calibrationTarget.material { roughness = 1.0f metallic = 0.0f diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.camera!!.addChild(calibrationTarget) + sciview.camera?.addChild(calibrationTarget) laser.visible = false laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) + laser.name = "Laser" + sciview.addNode(laser) val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) shell.ifMaterial{ @@ -120,9 +121,8 @@ class EyeTrackingDemo: Command{ diffuse = Vector3f(0.4f, 0.4f, 0.4f) } shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) + sciview.addNode(shell) -// volume = sciview.find("volume") as Volume val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } val v = (volnodes.firstOrNull() as? Volume) @@ -139,18 +139,18 @@ class EyeTrackingDemo: Command{ bb.node = volume bb.visible = false - sciview.addChild(hedgehogs) + sciview.addNode(hedgehogs) val eyeFrames = Mesh("eyeFrames") val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) - left.spatial().rotation = left.rotation.rotationZ(PI.toFloat()) + left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) eyeFrames.addChild(left) eyeFrames.addChild(right) - sciview.addChild(eyeFrames) + sciview.addNode(eyeFrames) val pupilFrameLimit = 20 var lastFrame = System.nanoTime() @@ -191,7 +191,7 @@ class EyeTrackingDemo: Command{ sciview.camera?.addChild(debugBoard) val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } + lights.forEach { sciview.addNode(it) } thread { log.info("Adding onDeviceConnect handlers") @@ -268,8 +268,8 @@ class EyeTrackingDemo: Command{ log.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) hedgehog.visible = false - hedgehog.setMaterial(ShaderMaterial.fromClass(ParticleDemo::class.java)) - var hedgehogInstanced = InstancedNode(hedgehog) + hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) + val hedgehogInstanced = InstancedNode(hedgehog) hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } hedgehogs.addChild(hedgehogInstanced) @@ -294,8 +294,8 @@ class EyeTrackingDemo: Command{ } val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) + val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) when(hedgehogVisibility) { HedgehogVisibility.Hidden -> { @@ -330,7 +330,7 @@ class EyeTrackingDemo: Command{ cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f, centered = true) } else { volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) + volume.spatial().scale =Vector3f(1.0f) .mul(volumeScaleFactor) } } @@ -340,7 +340,7 @@ class EyeTrackingDemo: Command{ cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 2f, size = 0.2f, centered = true) } else { volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) + volume.spatial().scale = Vector3f(1.0f) .mul(volumeScaleFactor) } } @@ -426,7 +426,6 @@ class EyeTrackingDemo: Command{ } - private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { val startCalibration = ClickBehaviour { _, _ -> thread { @@ -494,11 +493,11 @@ class EyeTrackingDemo: Command{ val p = gaze.gazePoint() referenceTarget.visible = true // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.position = p + referenceTarget.spatial().position = p (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" - val headCenter = cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.world).transform(p.xyzw()).xyz() + val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() val direction = (pointWorld - headCenter).normalize() if (tracking) { @@ -521,6 +520,7 @@ class EyeTrackingDemo: Command{ hmd.addBehaviour("start_calibration", startCalibration) hmd.addKeyBinding("start_calibration", keybindingCalibration) } + fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { val cam = sciview.camera as? DetachedHeadCamera ?: return val sphere = volume.boundingBox?.getBoundingSphere() ?: return @@ -556,7 +556,7 @@ class EyeTrackingDemo: Command{ localDirection, cam.headPosition, cam.headOrientation, - cam.position, + cam.spatial().position, confidence, samples.map { it ?: 0.0f } ) @@ -607,7 +607,7 @@ class EyeTrackingDemo: Command{ val track = if(existingAnalysis is HedgehogAnalysis.Track) { existingAnalysis } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.world)) + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) h.run() } @@ -640,13 +640,11 @@ class EyeTrackingDemo: Command{ trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> - if(mainTrack != null) { - val element = mainTrack.addInstance() - element.addAttribute(Material::class.java, cylinder.material()) - element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) - element.parent = volume + val element = mainTrack.addInstance() + element.addAttribute(Material::class.java, cylinder.material()) + element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) + element.parent = volume // mainTrack.instances.add(element) - } val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") From 7b8031d99e93193cc8c74332619fe5550cbdc225 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:46:07 +0200 Subject: [PATCH 38/55] Make eye tracking compatible with mastodon bridge --- .../commands/demo/advanced/EyeTrackingDemo.kt | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 6e0ba053..92fa99b5 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -39,7 +39,6 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import sc.iview.commands.demo.animation.ParticleDemo @Plugin(type = Command::class, menuRoot = "SciView", @@ -53,6 +52,12 @@ class EyeTrackingDemo: Command{ @Parameter private lateinit var log: LogService + @Parameter + private lateinit var mastodonCallbackLinkCreate: (HedgehogAnalysis.SpineGraphVertex) -> Unit + + @Parameter + private lateinit var mastodonUpdateGraph: () -> Unit + val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) lateinit var hmd: OpenVRHMD val referenceTarget = Icosphere(0.004f, 2) @@ -622,16 +627,16 @@ class EyeTrackingDemo: Command{ // logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) - cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { - diffuse = Vector3f(1f) - ambient = Vector3f(1f) - roughness = 1f - } +// val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) +// cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { +// diffuse = Vector3f(1f) +// ambient = Vector3f(1f) +// roughness = 1f +// } - cylinder.name = "Track-$hedgehogId" - val mainTrack = InstancedNode(cylinder) - mainTrack.instancedProperties["Color"] = { Vector4f(1f) } +// cylinder.name = "Track-$hedgehogId" +// val mainTrack = InstancedNode(cylinder) +// mainTrack.instancedProperties["Color"] = { Vector4f(1f) } val parentId = 0 val volumeDimensions = volume.getDimensions() @@ -640,17 +645,19 @@ class EyeTrackingDemo: Command{ trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") track.points.windowed(2, 1).forEach { pair -> - val element = mainTrack.addInstance() - element.addAttribute(Material::class.java, cylinder.material()) - element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) - element.parent = volume + mastodonCallbackLinkCreate(pair[0].second) +// val element = mainTrack.addInstance() +// element.addAttribute(Material::class.java, cylinder.material()) +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) +// element.parent = volume // mainTrack.instances.add(element) val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } + mastodonUpdateGraph() - mainTrack.let { sciview.addNode(it, parent = volume) } +// mainTrack.let { sciview.addNode(it, parent = volume) } trackFileWriter.close() } From 153dabf60b142e4ee0899ce3949618260a3718b0 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:01:47 +0200 Subject: [PATCH 39/55] Change log to logger for consistency --- .../commands/demo/advanced/EyeTrackingDemo.kt | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 92fa99b5..27434352 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -50,7 +50,7 @@ class EyeTrackingDemo: Command{ private lateinit var sciview: SciView @Parameter - private lateinit var log: LogService + private lateinit var logger: LogService @Parameter private lateinit var mastodonCallbackLinkCreate: (HedgehogAnalysis.SpineGraphVertex) -> Unit @@ -95,7 +95,7 @@ class EyeTrackingDemo: Command{ override fun run() { sciview.toggleVRRendering() - log.info("VR mode has been toggled") + logger.info("VR mode has been toggled") hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) @@ -132,10 +132,10 @@ class EyeTrackingDemo: Command{ val v = (volnodes.firstOrNull() as? Volume) if(v == null) { - log.warn("No volume found, bailing") + logger.warn("No volume found, bailing") return } else { - log.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") + logger.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") volume = v } volume.visible = false @@ -199,17 +199,17 @@ class EyeTrackingDemo: Command{ lights.forEach { sciview.addNode(it) } thread { - log.info("Adding onDeviceConnect handlers") + logger.info("Adding onDeviceConnect handlers") hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") + logger.info("onDeviceConnect called, cam=${sciview.camera}") if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") + logger.info("Got device ${device.name} at $timestamp") device.model?.let { hmd.attachToNode(device, it, sciview.camera) } } } } thread{ - log.info("started thread for inputSetup") + logger.info("started thread for inputSetup") inputSetup() } thread { @@ -270,7 +270,7 @@ class EyeTrackingDemo: Command{ } fun addHedgehog() { - log.info("added hedgehog") + logger.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) hedgehog.visible = false hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) @@ -426,7 +426,7 @@ class EyeTrackingDemo: Command{ hmd.addKeyBinding("cell_division", "T") hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - log.info("calibration should start now") + logger.info("calibration should start now") setupCalibration() } @@ -437,7 +437,7 @@ class EyeTrackingDemo: Command{ val cam = sciview.camera as? DetachedHeadCamera ?: return@thread pupilTracker.gazeConfidenceThreshold = confidenceThreshold if (!pupilTracker.isCalibrated) { - log.info("pupil is currently uncalibrated") + logger.info("pupil is currently uncalibrated") pupilTracker.onCalibrationInProgress = { cam.showMessage("Crunching equations ...",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000, centered = true) } @@ -462,12 +462,12 @@ class EyeTrackingDemo: Command{ val toggleTracking = ClickBehaviour { _, _ -> if (tracking) { - log.info("deactivating tracking...") + logger.info("deactivating tracking...") referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) dumpHedgehog() } else { - log.info("activating tracking...") + logger.info("activating tracking...") addHedgehog() referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) @@ -485,7 +485,7 @@ class EyeTrackingDemo: Command{ pupilTracker.unsubscribeFrames() sciview.deleteNode(sciview.find("eyeFrames")) - log.info("Starting eye tracker calibration") + logger.info("Starting eye tracker calibration") cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) pupilTracker.calibrate(cam, hmd, generateReferenceData = true, @@ -515,7 +515,7 @@ class EyeTrackingDemo: Command{ // else -> {gaze-> } } - log.info("Calibration routine done.") + logger.info("Calibration routine done.") } // bind calibration start to menu key on controller @@ -541,8 +541,7 @@ class EyeTrackingDemo: Command{ spine.spatial().orientBetweenPoints(p1, p2, true, true) spine.visible = true - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) -// System.out.println(intersection); + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) if(intersection is MaybeIntersects.Intersection) { // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) @@ -582,7 +581,7 @@ class EyeTrackingDemo: Command{ * If [hedgehog] is not null, the cell track will not be added to the scene. */ fun dumpHedgehog() { - log.info("dumping hedgehog...") + logger.info("dumping hedgehog...") val lastHedgehog = hedgehogs.children.last() as InstancedNode val hedgehogId = hedgehogIds.incrementAndGet() From b03c042a4329d09833b897da8f8961d0a4eba955 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:40:11 +0200 Subject: [PATCH 40/55] Add: gaussian smoothing function --- .../demo/advanced/HedgehogAnalysis.kt | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 645019f8..9f956266 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -7,7 +7,6 @@ import graphics.scenery.utils.extensions.* import graphics.scenery.utils.lazyLogger import org.slf4j.LoggerFactory import java.io.File -import kotlin.math.log import kotlin.math.sqrt /** @@ -56,14 +55,14 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix * packaged nicely as a Pair */ private fun localMaxima(list: List): List> { - return list.windowed(6, 2).mapIndexed { index, l -> + return list.windowed(3, 1).mapIndexed { index, l -> val left = l[0] - val center = l[2] - val right = l[4] + val center = l[1] + val right = l[2] // we have a match at center if (left < center && center > right) { - index * 2 + 2 to center + index * 1 + 1 to center } else { null } @@ -137,6 +136,25 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix logger.info("${timepoints.size} timepoints left") + fun gaussSmoothing(samples: List, iterations: Int): List { + var smoothed = samples.toList() + val kernel = listOf(0.25f, 0.5f, 0.25f) + for (i in 0 until iterations) { + val newSmoothed = ArrayList(smoothed.size) + // Handle the first element + newSmoothed.add(smoothed[0] * 0.75f + smoothed[1] * 0.25f) + // Apply smoothing to the middle elements + for (j in 1 until smoothed.size - 1) { + val value = kernel[0] * smoothed[j-1] + kernel[1] * smoothed[j] + kernel[2] * smoothed[j+1] + newSmoothed.add(value) + } + // Handle the last element + newSmoothed.add(smoothed[smoothed.size - 2] * 0.25f + smoothed[smoothed.size - 1] * 0.75f) + + smoothed = newSmoothed + } + return smoothed + } //step2: find the maxIndices along the spine // yo dawg, this will be a list of lists, where each entry in the first-level list From b5e5edae6a0a7bd9cfbbbb44450df55f17794ee4 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:41:57 +0200 Subject: [PATCH 41/55] Change to sampleRayGridTraversal and add samplePos list to SpineMetadata --- .../commands/demo/advanced/EyeTrackingDemo.kt | 27 ++++++++++++++++--- .../commands/demo/advanced/SpineMetadata.kt | 3 ++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 27434352..58770f98 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -547,9 +547,27 @@ class EyeTrackingDemo: Command{ val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) // TODO: Allow for sampling a given time point of a volume - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) - - if (samples != null && localDirection != null) { +// val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) + // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now + val localDirection = Vector3f(0f) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + +// if (samples != null && localDirection != null) { +// val metadata = SpineMetadata( +// timepoint, +// center, +// direction, +// intersection.distance, +// localEntry, +// localExit, +// localDirection, +// cam.headPosition, +// cam.headOrientation, +// cam.spatial().position, +// confidence, +// samples.map { it ?: 0.0f } +// ) + if (samples != null && samplePos != null) { val metadata = SpineMetadata( timepoint, center, @@ -562,7 +580,8 @@ class EyeTrackingDemo: Command{ cam.headOrientation, cam.spatial().position, confidence, - samples.map { it ?: 0.0f } + samples.map { it ?: 0.0f }, + samplePos.map { it ?: Vector3f(0f) } ) val count = samples.filterNotNull().count { it > 0.2f } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt index 5192d096..cbbaf4a4 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/SpineMetadata.kt @@ -19,5 +19,6 @@ data class SpineMetadata( // val headOrientation: Quaternion, val position: Vector3f, val confidence: Float, - val samples: List + val samples: List, + val samplePosList: List = ArrayList() ) \ No newline at end of file From 1d77ea366d7c8b94c26d859aff8773538d8f2231 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:32:24 +0200 Subject: [PATCH 42/55] HedgehogAnalysis: get spine vertex position from grid traversal --- .../kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 9f956266..10c79178 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -173,7 +173,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix //filter { it.first <1200}. maxIndices.map { index -> logger.debug("Generating vertex at index $index") - val position = Vector3f(spine.localEntry).add((Vector3f(spine.localDirection).mul(index.first.toFloat()))) + val position = spine.position val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() SpineGraphVertex(tp.key, position, From 34b76f816316153a4fdaf972608d691a8a04aa64 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:01:12 +0200 Subject: [PATCH 43/55] Fix: ignore children in AABB intersection --- .../kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 58770f98..00c3814c 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -541,7 +541,7 @@ class EyeTrackingDemo: Command{ spine.spatial().orientBetweenPoints(p1, p2, true, true) spine.visible = true - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) if(intersection is MaybeIntersects.Intersection) { // get local entry and exit coordinates, and convert to UV coords val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) From 7325742698c0dcebba4156c3b15cc27495c5eceb Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:58:04 +0200 Subject: [PATCH 44/55] EyeTrackingDemo: fix vertex positions --- .../sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 5 +++-- .../sc/iview/commands/demo/advanced/HedgehogAnalysis.kt | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 00c3814c..267c8507 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -39,6 +39,7 @@ import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import graphics.scenery.volumes.RAIVolume @Plugin(type = Command::class, menuRoot = "SciView", @@ -551,7 +552,7 @@ class EyeTrackingDemo: Command{ // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now val localDirection = Vector3f(0f) val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) - + val volumeScale = (volume as RAIVolume).getVoxelScale() // if (samples != null && localDirection != null) { // val metadata = SpineMetadata( // timepoint, @@ -581,7 +582,7 @@ class EyeTrackingDemo: Command{ cam.spatial().position, confidence, samples.map { it ?: 0.0f }, - samplePos.map { it ?: Vector3f(0f) } + samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } ) val count = samples.filterNotNull().count { it > 0.2f } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt index 10c79178..bf183131 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/HedgehogAnalysis.kt @@ -157,7 +157,7 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix } //step2: find the maxIndices along the spine - // yo dawg, this will be a list of lists, where each entry in the first-level list + // this will be a list of lists, where each entry in the first-level list // corresponds to a time point, which then contains a list of vertices within that timepoint. val candidates: List> = timepoints.map { tp -> val vs = tp.value.mapIndexedNotNull { i, spine -> @@ -173,7 +173,8 @@ class HedgehogAnalysis(val spines: List, val localToWorld: Matrix //filter { it.first <1200}. maxIndices.map { index -> logger.debug("Generating vertex at index $index") - val position = spine.position + // get the position of the current index along the spine + val position = spine.samplePosList[index.first] val worldPosition = localToWorld.transform((Vector3f(position)).xyzw()).xyz() SpineGraphVertex(tp.key, position, From f51de82e31ac547a0932d2db8b5ffe598ad4eff7 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:21:39 +0200 Subject: [PATCH 45/55] Create CellTrackingBase, let other classes inherit most of the (otherwise duplicated) methods --- .../commands/demo/advanced/EyeTrackingDemo.kt | 408 +--------------- .../sc/iview/commands/demo/advanced/Test.kt | 229 +-------- .../demo/advanced/VRControllerTrackingDemo.kt | 395 +-------------- .../demo/advanced/VRHeadSetTrackingDemo.kt | 428 +---------------- .../sc/iview/process/CellTrackingBase.kt | 448 ++++++++++++++++++ 5 files changed, 494 insertions(+), 1414 deletions(-) create mode 100644 src/main/kotlin/sc/iview/process/CellTrackingBase.kt diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 267c8507..5fa3c44e 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -40,59 +40,32 @@ import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import graphics.scenery.volumes.RAIVolume +import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class EyeTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView +class EyeTrackingDemo: Command, CellTrackingBase() { @Parameter - private lateinit var logger: LogService + override var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null @Parameter - private lateinit var mastodonCallbackLinkCreate: (HedgehogAnalysis.SpineGraphVertex) -> Unit + override var mastodonUpdateGraph: (() -> Unit)? = null + - @Parameter - private lateinit var mastodonUpdateGraph: () -> Unit val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) + val calibrationTarget = Icosphere(0.02f, 2) val laser = Cylinder(0.005f, 0.2f, 10) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - val confidenceThreshold = 0.60f - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 - var skipToNext = false - var skipToPrevious = false // var currentVolume = 0 - var volumeScaleFactor = 1.0f - override fun run() { sciview.toggleVRRendering() @@ -212,226 +185,13 @@ class EyeTrackingDemo: Command{ thread{ logger.info("started thread for inputSetup") inputSetup() + setupCalibration() } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hh -> - val hedgehog = hh as InstancedNode - hedgehog.instances.forEach { - if (it.metadata.isNotEmpty()) { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } else { - hedgehogs.children.forEach { hh -> - val hedgehog = hh as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - fun addHedgehog() { - logger.info("added hedgehog") - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false - hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) - val hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) + launchHedgehogThread() } - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogs.visible = false - hedgehogs.runRecursive { it.visible = false } - cam.showMessage("Hedgehogs hidden",distance = 2f, size = 0.2f, centered = true) - } - - HedgehogVisibility.PerTimePoint -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs shown per timepoint",distance = 2f, size = 0.2f, centered = true) - } - - HedgehogVisibility.Visible -> { - hedgehogs.visible = true - cam.showMessage("Hedgehogs visible",distance = 2f, size = 0.2f, centered = true) - } - } - } - - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f, centered = true) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.spatial().scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 2f, size = 0.2f, centered = true) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.spatial().scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 2f, size = 0.2f, centered = true) - } else { - cam.showMessage("Paused",distance = 2f, size = 0.2f, centered = true) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt(), - centered = true) - - }, - confirmAction = { - hedgehogs.children.removeLast() - volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> - volume.removeChild(lastTrack) - } - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000, - centered = true - ) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 2f, centered = true) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 2f, duration = 1000) - dumpHedgehog() - addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - logger.info("calibration should start now") - setupCalibration() - - } - private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { val startCalibration = ClickBehaviour { _, _ -> thread { @@ -527,159 +287,7 @@ class EyeTrackingDemo: Command{ hmd.addKeyBinding("start_calibration", keybindingCalibration) } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = sphere.origin.minus(center) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = center - val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - val spine = (hedgehogs.children.last() as InstancedNode).addInstance() - spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = true - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - // TODO: Allow for sampling a given time point of a volume -// val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) - // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now - val localDirection = Vector3f(0f) - val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) - val volumeScale = (volume as RAIVolume).getVoxelScale() -// if (samples != null && localDirection != null) { -// val metadata = SpineMetadata( -// timepoint, -// center, -// direction, -// intersection.distance, -// localEntry, -// localExit, -// localDirection, -// cam.headPosition, -// cam.headOrientation, -// cam.spatial().position, -// confidence, -// samples.map { it ?: 0.0f } -// ) - if (samples != null && samplePos != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.spatial().position, - confidence, - samples.map { it ?: 0.0f }, - samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } - ) - val count = samples.filterNotNull().count { it > 0.2f } - - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - // TODO: Show confidence as color for the spine - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - logger.info("dumping hedgehog...") - val lastHedgehog = hedgehogs.children.last() as InstancedNode - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - -// val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) -// cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { -// diffuse = Vector3f(1f) -// ambient = Vector3f(1f) -// roughness = 1f -// } - -// cylinder.name = "Track-$hedgehogId" -// val mainTrack = InstancedNode(cylinder) -// mainTrack.instancedProperties["Color"] = { Vector4f(1f) } - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - mastodonCallbackLinkCreate(pair[0].second) -// val element = mainTrack.addInstance() -// element.addAttribute(Material::class.java, cylinder.material()) -// element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) -// element.parent = volume -// mainTrack.instances.add(element) - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - mastodonUpdateGraph() - -// mainTrack.let { sciview.addNode(it, parent = volume) } - - trackFileWriter.close() - } companion object { diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index c57150ba..347a643d 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -20,6 +20,7 @@ import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin import sc.iview.SciView import sc.iview.commands.MenuWeights +import sc.iview.process.CellTrackingBase import java.io.File import java.nio.file.Files import java.nio.file.Path @@ -33,55 +34,25 @@ import kotlin.concurrent.thread menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class Test: Command{ - @Parameter - private lateinit var sciview: SciView +class Test: Command, CellTrackingBase() { - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.004f, 2) //val calibrationTarget = Icosphere(0.02f, 2) val TestTarget = Icosphere(0.1f, 2) val laser = Cylinder(0.005f, 0.2f, 10) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path lateinit var point1:Icosphere lateinit var point2:Icosphere - - val hedgehogs = Mesh() - - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - val confidenceThreshold = 0.60f - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Backward - @Parameter(label = "Volumes per second") - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false // var currentVolume = 0 - var volumeScaleFactor = 1.0f - override fun run() { - sciview.addChild(TestTarget) + sciview.addNode(TestTarget) TestTarget.visible = false - // sciview.toggleVRRendering() // hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" @@ -97,7 +68,7 @@ class Test: Command{ laser.visible = false laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(laser) + sciview.addNode(laser) val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) shell.ifMaterial{ @@ -105,7 +76,7 @@ class Test: Command{ diffuse = Vector3f(0.4f, 0.4f, 0.4f) } shell.name = "shell" shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - sciview.addChild(shell) + sciview.addNode(shell) volume = sciview.find("volume") as Volume volume.visible = false @@ -113,113 +84,48 @@ class Test: Command{ point1 = Icosphere(0.1f, 2) point1.spatial().position = Vector3f(1.858f,2f,8.432f) point1.ifMaterial{ diffuse = Vector3f(0.5f, 0.3f, 0.8f)} - sciview.addChild(point1) + sciview.addNode(point1) point2 = Icosphere(0.1f, 2) point2.spatial().position = Vector3f(1.858f, 2f, -10.39f) point2.ifMaterial {diffuse = Vector3f(0.3f, 0.8f, 0.3f)} - sciview.addChild(point2) + sciview.addNode(point2) - val connector = Cylinder.betweenPoints(point1.position, point2.position) + val connector = Cylinder.betweenPoints(point1.spatial().position, point2.spatial().position) connector.ifMaterial {diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - sciview.addChild(connector) + sciview.addNode(connector) val bb = BoundingGrid() bb.node = volume bb.visible = false - sciview.addChild(hedgehogs) + sciview.addNode(hedgehogs) val pupilFrameLimit = 20 var lastFrame = System.nanoTime() - - val debugBoard = TextBoard() debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) debugBoard.text = "" debugBoard.visible = false sciview.camera?.addChild(debugBoard) val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addChild(it) } + lights.forEach { sciview.addNode(it) } thread{ inputSetup() } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - //println("timepoint: "+ newTimepoint); - - if(hedgehogs.visible) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogs.children.forEach { hedgehog-> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } else { - hedgehogs.children.forEach { hedgehog -> - val hedgehog = hedgehog as InstancedNode - hedgehog.instances.forEach { it.visible = true } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - //dumpHedgehog() - } - } - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehog = Cylinder(0.005f, 1.0f, 16) - hedgehog.visible = false -// hedgehog.material = ShaderMaterial.fromClass(BionicTracking::class.java, -// listOf(ShaderType.VertexShader, ShaderType.FragmentShader)) - var hedgehogInstanced = InstancedNode(hedgehog) - hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} - hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } - hedgehogs.addChild(hedgehogInstanced) + launchHedgehogThread() } - - fun inputSetup() + override fun inputSetup() { val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") setupControllerforTracking() @@ -240,8 +146,8 @@ class Test: Command{ // val p = Vector3f(0f,0f,-1f) // referenceTarget.position = p // referenceTarget.visible = true - val headCenter = point1.position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = point2.position///Matrix4f(cam.world).transform(p.xyzw()).xyz() + val headCenter = point1.spatial().position//cam.viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = point2.spatial().position///Matrix4f(cam.world).transform(p.xyzw()).xyz() // val direction = (pointWorld - headCenter).normalize() @@ -326,18 +232,15 @@ class Test: Command{ } } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + override fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { val cam = sciview.camera as? DetachedHeadCamera ?: return val sphere = volume.boundingBox?.getBoundingSphere() ?: return val sphereDirection = sphere.origin.minus(center) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius val p1 = center val temp = direction.mul(sphereDist + 2.0f * sphere.radius) - - val p2 = Vector3f(center).add(temp) @@ -369,7 +272,7 @@ class Test: Command{ // System.out.println("worldExit:" + worldpositionExit.toString()) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null + val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) if (samples != null && localDirection != null) { val metadata = SpineMetadata( @@ -388,8 +291,8 @@ class Test: Command{ ) val count = samples.filterNotNull().count { it > 0.002f } - println("count of samples: "+ count.toString()) -println(samples) + logger.info("count of samples: "+ count.toString()) + logger.info(samples) // spine.metadata["spine"] = metadata // spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } @@ -398,96 +301,6 @@ println(samples) } } - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ -// fun dumpHedgehog() { -// var lastHedgehog = hedgehogs.children.last() as InstancedNode -// val hedgehogId = hedgehogIds.incrementAndGet() -// -// val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() -// val hedgehogFileWriter = hedgehogFile.bufferedWriter() -// hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") -// -// val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() -// val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) -// if(!trackFile.exists()) { -// trackFile.createNewFile() -// trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") -// trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") -// } -// -// -// val spines = lastHedgehog.instances.mapNotNull { spine -> -// spine.metadata["spine"] as? SpineMetadata -// } -// -// spines.forEach { metadata -> -// hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") -// } -// hedgehogFileWriter.close() -// -// val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track -// val track = if(existingAnalysis is HedgehogAnalysis.Track) { -// existingAnalysis -// } else { -// val h = HedgehogAnalysis(spines, Matrix4f(volume.world), Vector3f(volume.getDimensions())) -// h.run() -// } -// -// if(track == null) { -//// logger.warn("No track returned") -// sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) -// return -// } -// -// lastHedgehog.metadata["HedgehogAnalysis"] = track -// lastHedgehog.metadata["Spines"] = spines -// -//// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") -// -// val master = if(lastHedgehog == null) { -// val m = Cylinder(3f, 1.0f, 10) -// m.ifMaterial { -// ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag") -// diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) -// roughness = 1.0f -// metallic = 0.0f -// cullingMode = Material.CullingMode.None -// } -// m.name = "Track-$hedgehogId" -// val mInstanced = InstancedNode(m) -// mInstanced -// } else { -// null -// } -// -// val parentId = 0 -// val volumeDimensions = volume.getDimensions() -// -// trackFileWriter.newLine() -// trackFileWriter.newLine() -// trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") -// track.points.windowed(2, 1).forEach { pair -> -// if(master != null) { -// val element = master.addInstance() -// element.spatial().orientBetweenPoints(Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)), Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)), rescale = true, reposition = true) -// element.parent = volume -// master.instances.add(element) -// } -// val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product -// val tp = pair[0].second.timepoint -// trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") -// } -// -// master?.let { volume.addChild(it) } -// -// trackFileWriter.close() -// } - companion object { @Throws(Exception::class) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index c3099ff6..241f1cf4 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -1,9 +1,6 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* -import graphics.scenery.controls.behaviours.ControllerDrag -import graphics.scenery.numerics.Random -import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz @@ -18,64 +15,34 @@ import org.scijava.plugin.Plugin import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.io.BufferedWriter -import java.io.FileWriter import java.nio.file.Files -import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.controls.* import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRControllerTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView +class VRControllerTrackingDemo: Command, CellTrackingBase() { - @Parameter - private lateinit var log: LogService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.04f, 2) val testTarget1 = Icosphere(0.01f, 2) val testTarget2 = Icosphere(0.04f, 2) val laser = Cylinder(0.0025f, 1f, 20) - lateinit var sessionId: String - lateinit var sessionDirectory: Path lateinit var rightController: TrackedDevice var hedgehogsList = mutableListOf() - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume - - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = false - var direction = PlaybackDirection.Forward - var volumesPerSecond = 4 - var skipToNext = false - var skipToPrevious = false // var currentVolume = 0 - var volumeScaleFactor = 1.0f - override fun run() { sciview.toggleVRRendering() @@ -143,11 +110,11 @@ class VRControllerTrackingDemo: Command{ lights.forEach { sciview.addChild(it) } thread { - log.info("Adding onDeviceConnect handlers") + logger.info("Adding onDeviceConnect handlers") hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") + logger.info("onDeviceConnect called, cam=${sciview.camera}") if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") + logger.info("Got device ${device.name} at $timestamp") // if(device.role == TrackerRole.RightHand) { // rightController = device // log.info("rightController is found, its location is in ${rightController.position}") @@ -159,222 +126,10 @@ class VRControllerTrackingDemo: Command{ } thread{ inputSetup() - } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - dumpHedgehog() - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) - } - - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward_fast" to "K", - "move_back_fast" to "J", - "move_left_fast" to "H", - "move_right_fast" to "L").forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } + setupControllerforTracking() } - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) - volume.scale =Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) - volume.scale = Vector3f(1.0f) .mul(volumeScaleFactor) - } - } - - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } - - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - - }, - confirmAction = { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - }) - - hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - }) - - val cellDivision = ClickBehaviour { _, _ -> - cam.showMessage("Adding cell division", distance = 1.2f, size = 0.2f, duration = 1000) - //dumpHedgehog() - //addHedgehog() - } - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - hmd.addBehaviour("trigger_move", move) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("cell_division", cellDivision) - - hmd.addKeyBinding("toggle_hedgehog", "X") - hmd.addKeyBinding("delete_hedgehog", "Y") - hmd.addKeyBinding("skip_to_next", "D") - hmd.addKeyBinding("skip_to_prev", "A") - hmd.addKeyBinding("faster_or_scale", "W") - hmd.addKeyBinding("slower_or_scale", "S") - hmd.addKeyBinding("play_pause", "M") - hmd.addKeyBinding("playback_direction", "N") - hmd.addKeyBinding("cell_division", "T") - - hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand - - setupControllerforTracking() - + launchHedgehogThread() } private fun setupControllerforTracking( keybindingTracking: String = "U") { @@ -404,7 +159,6 @@ class VRControllerTrackingDemo: Command{ volume.runRecursive { it.visible = true } playing = true - while(true) { /** @@ -454,141 +208,6 @@ class VRControllerTrackingDemo: Command{ } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.intersectAABB(p1, (p2 - p1).normalize()) - - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: null to null - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - localEntry, - localExit, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - val count = samples.filterNotNull().count { it > 0.02f } - //println("cnt: " + count.toString()) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } -// spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - if(track == null) { - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - - val master = Cylinder(0.1f, 1.0f, 10) - master.setMaterial (ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - - master.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - master.name = "Track-$hedgehogId" - val mInstanced = InstancedNode(master) - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - sciview.addNode(mInstanced) - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = mInstanced.addInstance() - val p0 = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val p1 = Vector3f(pair[1].first).mul(Vector3f(volumeDimensions)) - val p0w = Matrix4f(volume.spatial().world).transform(p0.xyzw()).xyz() - val p1w = Matrix4f(volume.spatial().world).transform(p1.xyzw()).xyz() - element.spatial().orientBetweenPoints(p0w,p1w, rescale = true, reposition = true) - val pp = Icosphere(0.01f, 1) - pp.spatial().position = p0w - pp.material().diffuse = Vector3f(0.5f, 0.3f, 0.8f) - sciview.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - val tp = pair[0].second.timepoint - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - trackFileWriter.close() - } companion object { diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index eb4a6342..fd9bfab5 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -4,8 +4,6 @@ import graphics.scenery.* import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole -import graphics.scenery.numerics.Random -import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.volumes.Volume import org.joml.* @@ -17,71 +15,37 @@ import org.scijava.plugin.Plugin import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.io.BufferedWriter import java.io.File -import java.io.FileWriter import java.nio.file.Files -import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.controls.behaviours.* -import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard import graphics.scenery.utils.extensions.* import org.scijava.event.EventService import sc.iview.commands.file.OpenDirofTif -import sc.iview.event.NodeAddedEvent -import sc.iview.event.NodeChangedEvent import sc.iview.event.NodeTaggedEvent +import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRHeadSetTrackingDemo: Command{ - @Parameter - private lateinit var sciview: SciView +class VRHeadSetTrackingDemo: Command, CellTrackingBase() { @Parameter - private lateinit var log: LogService - - @Parameter - private lateinit var eventService: EventService - - lateinit var hmd: OpenVRHMD - val referenceTarget = Icosphere(0.02f, 2) - - lateinit var sessionId: String - lateinit var sessionDirectory: Path + private lateinit var eventService: EventService var hedgehogsList = mutableListOf() - enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } - var hedgehogVisibility = HedgehogVisibility.Hidden - - lateinit var volume: Volume private var selectionStorage: Node? = null - enum class PlaybackDirection { - Forward, - Backward - } - - @Volatile var tracking = false - var playing = true - var direction = PlaybackDirection.Backward - var volumesPerSecond = 1 - var skipToNext = false - var skipToPrevious = false // var currentVolume = 0 - var volumeScaleFactor = 1.0f - override fun run() { sciview.toggleVRRendering() @@ -129,237 +93,26 @@ class VRHeadSetTrackingDemo: Command{ lights.forEach { sciview.addChild(it) } thread { - log.info("Adding onDeviceConnect handlers") + logger.info("Adding onDeviceConnect handlers") hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - log.info("onDeviceConnect called, cam=${sciview.camera}") + logger.info("onDeviceConnect called, cam=${sciview.camera}") if(device.type == TrackedDeviceType.Controller) { - log.info("Got device ${device.name} at $timestamp") + logger.info("Got device ${device.name} at $timestamp") device.model?.let { hmd.attachToNode(device, it, sciview.camera) } } } } thread{ inputSetup() + setupHeadsetTracking() } - thread { - while(!sciview.isInitialized) { Thread.sleep(200) } - - while(sciview.running) { - if(playing || skipToNext || skipToPrevious) { - val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { - skipToNext = false - if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() - } else { - volume.previousTimepoint() - } - } else { - skipToPrevious = false - if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() - } else { - volume.nextTimepoint() - } - } - val newTimepoint = volume.viewerState.currentTimepoint - - if(hedgehogsList.size>0) { - if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { - hedgehogsList.forEach { hedgehog-> - hedgehog.instances.forEach { - it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint - } - } - } - } - - if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { - tracking = false - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} - sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - thread { - dumpHedgehog() - } - } - } - - Thread.sleep((1000.0f/volumesPerSecond).toLong()) - } - } - - } - - fun addHedgehog() { - val hedgehogMaster = Cylinder(0.1f, 1.0f, 16) - hedgehogMaster.visible = false - hedgehogMaster.setMaterial(ShaderMaterial.fromFiles("DefaultDeferredInstanced.vert", "DefaultDeferred.frag")) - hedgehogMaster.ifMaterial{ - ambient = Vector3f(0.1f, 0f, 0f) - diffuse = Random.random3DVectorFromRange(0.2f, 0.8f) - metallic = 0.01f - roughness = 0.5f - } - var hedgehogInstanced = InstancedNode(hedgehogMaster) - sciview.addNode(hedgehogInstanced) - hedgehogsList.add(hedgehogInstanced) + launchHedgehogThread() } - - fun inputSetup() - { - val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") - - //set up move action for moving forward, back, left and right - sciview.sceneryInputHandler?.let { handler -> - hashMapOf( - "move_forward" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Up), - "move_back" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Down), - "move_left" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Left), - "move_right" to OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Right) - ).forEach { (name, key) -> - handler.getBehaviour(name)?.let { b -> - hmd.addBehaviour(name, b) - hmd.addKeyBinding(name, key) - } - } - } - - //hedgehog has three modes of visibility, 1. hide hedgehog, 2, show hedgehog for per timepoint, 3. show full hedgehog - val toggleHedgehog = ClickBehaviour { _, _ -> - val current = HedgehogVisibility.values().indexOf(hedgehogVisibility) - hedgehogVisibility = HedgehogVisibility.values().get((current + 1) % 3) - - when(hedgehogVisibility) { - HedgehogVisibility.Hidden -> { - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = false } - } - cam.showMessage("Hedgehogs hidden",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.PerTimePoint -> { - cam.showMessage("Hedgehogs shown per timepoint",distance = 1.2f, size = 0.2f) - } - - HedgehogVisibility.Visible -> { - println("the number of hedgehogs: "+ hedgehogsList.size.toString()) - hedgehogsList.forEach { hedgehog -> - println("the number of spines: " + hedgehog.instances.size.toString()) - hedgehog.instances.forEach { it.visible = true } - } - cam.showMessage("Hedgehogs visible",distance = 1.2f, size = 0.2f) - } - } - } - - //adjust the direction of playing volume - val nextTimepoint = ClickBehaviour { _, _ -> - skipToNext = true - } - - val prevTimepoint = ClickBehaviour { _, _ -> - skipToPrevious = true - } - - //Speeding up the playing of volume or enlarge the scale of volume depending on whether the volume is in playing - val fasterOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = minOf(volumeScaleFactor * 1.1f, 1.2f) - volume.spatial().scale *= Vector3f(volumeScaleFactor) - } - } - - //slower the playing of volume or reduce the scale of volume depending on whether the volume is in play - val slowerOrScale = ClickBehaviour { _, _ -> - if(playing) { - volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) - cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f) - } else { - volumeScaleFactor = maxOf(volumeScaleFactor / 1.1f, 0.9f) - volume.spatial().scale *= Vector3f(volumeScaleFactor) - } - } - - //click the button to play or pause the volume - val playPause = ClickBehaviour { _, _ -> - playing = !playing - if(playing) { - cam.showMessage("Playing",distance = 1.2f, size = 0.2f) - } else { - cam.showMessage("Paused",distance = 1.2f, size = 0.2f) - } - } - - //delete the last hedgehog - val deleteLastHedgehog = ConfirmableClickBehaviour( - armedAction = { timeout -> - cam.showMessage("Deleting last track, press again to confirm.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - duration = timeout.toInt()) - }, - confirmAction = { - if(hedgehogsList.size != 0) - { - hedgehogsList = hedgehogsList.dropLast(1) as MutableList -// sciview.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> -// sciview.removeChild(lastTrack) -// } - val hedgehogId = hedgehogIds.get() - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) - hedgehogFileWriter.newLine() - hedgehogFileWriter.newLine() - hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") - hedgehogFileWriter.close() - - cam.showMessage("Last track deleted.",distance = 1.2f, size = 0.2f, - messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), - backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), - duration = 1000) - } - }) - - val playbackDirection = ClickBehaviour { _, _ -> - direction = if(direction == PlaybackDirection.Forward) { - PlaybackDirection.Backward - } else { - PlaybackDirection.Forward - } - cam.showMessage("Playing: ${direction}", distance = 1.2f, size = 0.2f, duration = 1000) - } - - - hmd.addBehaviour("skip_to_next", nextTimepoint) - hmd.addBehaviour("skip_to_prev", prevTimepoint) - hmd.addBehaviour("faster_or_scale", fasterOrScale) - hmd.addBehaviour("slower_or_scale", slowerOrScale) - hmd.addBehaviour("play_pause", playPause) - hmd.addBehaviour("playback_direction",playbackDirection) - hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) - hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) - - - - hmd.addKeyBinding("skip_to_next", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Right)) // RightController. right - hmd.addKeyBinding("skip_to_prev", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Left)) // RightController. left - hmd.addKeyBinding("faster_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Up)) // RightController. up - hmd.addKeyBinding("slower_or_scale", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Down)) //RightController. down - hmd.addKeyBinding("play_pause", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.Menu)) // LeftController.Menu - hmd.addKeyBinding("playback_direction", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.Menu)) //RightController.Menu - hmd.addKeyBinding("delete_hedgehog", OpenVRHMD.keyBinding(TrackerRole.RightHand,OpenVRHMD.OpenVRButton.A)) //RightController.Side - hmd.addKeyBinding("toggle_hedgehog", OpenVRHMD.keyBinding(TrackerRole.LeftHand,OpenVRHMD.OpenVRButton.A)) //LeftController.Side - - - + private fun setupHeadsetTracking() { //VRGrab.createAndSet(scene = Scene(), hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), listOf(TrackerRole.LeftHand)) - //left trigger button can validate or delete a track, the function should be arranged to two different button in the future + //left trigger button can validate or delete a track, the function should be arranged to two different button in the future VRSelect.createAndSet(sciview.currentScene, hmd, listOf(OpenVRHMD.OpenVRButton.Trigger), @@ -389,8 +142,6 @@ class VRHeadSetTrackingDemo: Command{ setupControllerforTracking() } - - private fun setupControllerforTracking( keybindingTracking: String = "U") { thread { val cam = sciview.camera as? DetachedHeadCamera ?: return@thread @@ -439,165 +190,6 @@ class VRHeadSetTrackingDemo: Command{ } } - fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { - val cam = sciview.camera as? DetachedHeadCamera ?: return - val sphere = volume.boundingBox?.getBoundingSphere() ?: return - - val sphereDirection = Vector3f(sphere.origin).minus(Vector3f(center)) - val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius - - val p1 = Vector3f(center) - val temp = Vector3f(direction).mul(sphereDist + 2.0f * sphere.radius) - val p2 = Vector3f(center).add(temp) - - var hedgehogsInstance = hedgehogsList.last() - val spine = hedgehogsInstance.addInstance() - spine.spatial().orientBetweenPoints(p1, p2,true,true) - spine.visible = false - - val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize()) - //println("try to find intersection"); - - if(intersection is MaybeIntersects.Intersection) { - // get local entry and exit coordinates, and convert to UV coords - val dim = volume.getDimensions() - - val entryUV = Vector3f(intersection.relativeEntry).div(Vector3f(dim)) - val exitUV = Vector3f(intersection.relativeExit).div(Vector3f(dim)) - val (samples, localDirection) = volume.sampleRay(entryUV, exitUV) ?: (null to null) - - - - if (samples != null && localDirection != null) { - val metadata = SpineMetadata( - timepoint, - center, - direction, - intersection.distance, - entryUV, - exitUV, - localDirection, - cam.headPosition, - cam.headOrientation, - cam.position, - confidence, - samples.map { it ?: 0.0f } - ) - spine.metadata["spine"] = metadata - spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } - } - } - } - - val hedgehogIds = AtomicInteger(0) - /** - * Dumps a given hedgehog including created tracks to a file. - * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. - * If [hedgehog] is not null, the cell track will not be added to the scene. - */ - fun dumpHedgehog() { - //println("size of hedgehogslist: " + hedgehogsList.size.toString()) - var lastHedgehog = hedgehogsList.last() - println("lastHedgehog: ${lastHedgehog}") - val hedgehogId = hedgehogIds.incrementAndGet() - - val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() - val hedgehogFileWriter = hedgehogFile.bufferedWriter() - hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") - - val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() - val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) - if(!trackFile.exists()) { - trackFile.createNewFile() - trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") - trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") - } - - - val spines = lastHedgehog.instances.mapNotNull { spine -> - spine.metadata["spine"] as? SpineMetadata - } - - spines.forEach { metadata -> - hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") - } - hedgehogFileWriter.close() - - val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track - val track = if(existingAnalysis is HedgehogAnalysis.Track) { - existingAnalysis - } else { - println("do hedgehog Analysis") - val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) - h.run() - } - - //check whether track is null, if it is null, then let the camera show "No track returned", otherwise do analysis - if(track == null) { -// logger.warn("No track returned") - sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) - return - } - - lastHedgehog.metadata["HedgehogAnalysis"] = track - lastHedgehog.metadata["Spines"] = spines - -// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") - - val parent = RichNode() - parent.name = "Track-$hedgehogId" - - val parentId = 0 - val volumeDimensions = volume.getDimensions() - - trackFileWriter.newLine() - trackFileWriter.newLine() - trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - track.points.windowed(2, 1).forEach { pair -> - val element = Cylinder(3.0f, 1.0f, 5)//edgeMaster.addInstance() - val p0 = Vector3f(pair[0].first) * Vector3f(volumeDimensions) - val p1 = Vector3f(pair[1].first) * Vector3f(volumeDimensions) - - val tp = pair[0].second.timepoint - - element.spatial().orientBetweenPoints(p0, p1, rescale = true, reposition = true) - element.name = "edge" - element.metadata["Type"] = "edge" - parent.addChild(element) - - val pp = Icosphere(5.0f, 1)//nodeMaster.addInstance() - log.info("Local position: $p0 / $p1") - pp.name = "node-$tp" - pp.metadata["NodeTimepoint"] = tp - pp.metadata["NodePosition"] = p0 - pp.metadata["Type"] = "node" - pp.spatial().position = p0 - - //give attributes to these nodes to make them grable, touchable and selectable, for more detailed usage check VRControllerExample in scenery - pp.addAttribute(Grabable::class.java, Grabable()) - pp.addAttribute(Selectable::class.java, Selectable(onSelect = {selectionStorage = pp})) - pp.addAttribute(Touchable::class.java, Touchable(onTouch = { device -> - if (device.role == TrackerRole.LeftHand) { - pp.ifSpatial { - position = (device.velocity ?: Vector3f(0.0f)) * 5f + position - eventService.publish(NodeChangedEvent(pp)) - } - } - - })) - parent.addChild(pp) - - val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions))//direct product - - trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") - } - - - volume.addChild(parent) - eventService.publish(NodeAddedEvent(parent)) - - trackFileWriter.close() - } companion object { //run function from here, it will automatically choose the volume for rendering, please give the correct location of volume diff --git a/src/main/kotlin/sc/iview/process/CellTrackingBase.kt b/src/main/kotlin/sc/iview/process/CellTrackingBase.kt new file mode 100644 index 00000000..80378e1e --- /dev/null +++ b/src/main/kotlin/sc/iview/process/CellTrackingBase.kt @@ -0,0 +1,448 @@ +package sc.iview.process + +import graphics.scenery.* +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.ControllerDrag +import graphics.scenery.primitives.Cylinder +import graphics.scenery.utils.MaybeIntersects +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.volumes.RAIVolume +import graphics.scenery.volumes.Volume +import org.joml.Math +import org.joml.Matrix4f +import org.joml.Vector3f +import org.joml.Vector4f +import org.scijava.log.LogService +import org.scijava.plugin.Parameter +import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView +import sc.iview.commands.demo.advanced.ConfirmableClickBehaviour +import sc.iview.commands.demo.advanced.HedgehogAnalysis +import sc.iview.commands.demo.advanced.SpineMetadata +import java.io.BufferedWriter +import java.io.FileWriter +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread + +open class CellTrackingBase { + + @Parameter + lateinit var sciview: SciView + + @Parameter + lateinit var logger: LogService + + lateinit var sessionId: String + lateinit var sessionDirectory: Path + + lateinit var hmd: OpenVRHMD + + val hedgehogs = Mesh() + val hedgehogIds = AtomicInteger(0) + lateinit var volume: Volume + + val referenceTarget = Icosphere(0.004f, 2) + + @Volatile var tracking = false + var playing = true + var direction = PlaybackDirection.Backward + var volumesPerSecond = 1 + var skipToNext = false + var skipToPrevious = false + + var volumeScaleFactor = 1.0f + + open var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null + open var mastodonUpdateGraph: (() -> Unit)? = null + + enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } + var hedgehogVisibility = HedgehogVisibility.Hidden + + enum class PlaybackDirection { + Forward, + Backward + } + + fun addHedgehog() { + logger.info("added hedgehog") + val hedgehog = Cylinder(0.005f, 1.0f, 16) + hedgehog.visible = false + hedgehog.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.frag", "DeferredInstancedColor.vert")) + val hedgehogInstanced = InstancedNode(hedgehog) + hedgehogInstanced.instancedProperties["ModelMatrix"] = { hedgehog.spatial().world} + hedgehogInstanced.instancedProperties["Metadata"] = { Vector4f(0.0f, 0.0f, 0.0f, 0.0f) } + hedgehogs.addChild(hedgehogInstanced) + } + + open fun inputSetup() + { + val cam = sciview.camera ?: throw IllegalStateException("Could not find camera") + + sciview.sceneryInputHandler?.let { handler -> + hashMapOf( + "move_forward_fast" to "K", + "move_back_fast" to "J", + "move_left_fast" to "H", + "move_right_fast" to "L").forEach { (name, key) -> + handler.getBehaviour(name)?.let { b -> + hmd.addBehaviour(name, b) + hmd.addKeyBinding(name, key) + } + } + } + + val toggleHedgehog = ClickBehaviour { _, _ -> + val current = HedgehogVisibility.entries.indexOf(hedgehogVisibility) + hedgehogVisibility = HedgehogVisibility.entries.get((current + 1) % 3) + + when(hedgehogVisibility) { + HedgehogVisibility.Hidden -> { + hedgehogs.visible = false + hedgehogs.runRecursive { it.visible = false } + cam.showMessage("Hedgehogs hidden",distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.PerTimePoint -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs shown per timepoint",distance = 2f, size = 0.2f, centered = true) + } + + HedgehogVisibility.Visible -> { + hedgehogs.visible = true + cam.showMessage("Hedgehogs visible",distance = 2f, size = 0.2f, centered = true) + } + } + } + + val nextTimepoint = ClickBehaviour { _, _ -> + skipToNext = true + } + + val prevTimepoint = ClickBehaviour { _, _ -> + skipToPrevious = true + } + + val fasterOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond+1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 1.2f, size = 0.2f, centered = true) + } else { + volumeScaleFactor = minOf(volumeScaleFactor * 1.2f, 3.0f) + volume.spatial().scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val slowerOrScale = ClickBehaviour { _, _ -> + if(playing) { + volumesPerSecond = maxOf(minOf(volumesPerSecond-1, 20), 1) + cam.showMessage("Speed: $volumesPerSecond vol/s",distance = 2f, size = 0.2f, centered = true) + } else { + volumeScaleFactor = maxOf(volumeScaleFactor / 1.2f, 0.1f) + volume.spatial().scale = Vector3f(1.0f) .mul(volumeScaleFactor) + } + } + + val playPause = ClickBehaviour { _, _ -> + playing = !playing + if(playing) { + cam.showMessage("Playing",distance = 2f, size = 0.2f, centered = true) + } else { + cam.showMessage("Paused",distance = 2f, size = 0.2f, centered = true) + } + } + + val move = ControllerDrag(TrackerRole.LeftHand, hmd) { volume } + + val deleteLastHedgehog = ConfirmableClickBehaviour( + armedAction = { timeout -> + cam.showMessage("Deleting last track, press again to confirm.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + backgroundColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + duration = timeout.toInt(), + centered = true) + + }, + confirmAction = { + hedgehogs.children.removeLast() + volume.children.last { it.name.startsWith("Track-") }?.let { lastTrack -> + volume.removeChild(lastTrack) + } + val hedgehogId = hedgehogIds.get() + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = BufferedWriter(FileWriter(hedgehogFile, true)) + hedgehogFileWriter.newLine() + hedgehogFileWriter.newLine() + hedgehogFileWriter.write("# WARNING: TRACK $hedgehogId IS INVALID\n") + hedgehogFileWriter.close() + + cam.showMessage("Last track deleted.",distance = 2f, size = 0.2f, + messageColor = Vector4f(1.0f, 0.2f, 0.2f, 1.0f), + backgroundColor = Vector4f(1.0f, 1.0f, 1.0f, 1.0f), + duration = 1000, + centered = true + ) + }) + + hmd.addBehaviour("playback_direction", ClickBehaviour { _, _ -> + direction = if(direction == PlaybackDirection.Forward) { + PlaybackDirection.Backward + } else { + PlaybackDirection.Forward + } + cam.showMessage("Playing: ${direction}", distance = 2f, centered = true) + }) + + val cellDivision = ClickBehaviour { _, _ -> + cam.showMessage("Adding cell division", distance = 2f, duration = 1000) + dumpHedgehog() + addHedgehog() + } + + hmd.addBehaviour("skip_to_next", nextTimepoint) + hmd.addBehaviour("skip_to_prev", prevTimepoint) + hmd.addBehaviour("faster_or_scale", fasterOrScale) + hmd.addBehaviour("slower_or_scale", slowerOrScale) + hmd.addBehaviour("play_pause", playPause) + hmd.addBehaviour("toggle_hedgehog", toggleHedgehog) + hmd.addBehaviour("trigger_move", move) + hmd.addBehaviour("delete_hedgehog", deleteLastHedgehog) + hmd.addBehaviour("cell_division", cellDivision) + + hmd.addKeyBinding("toggle_hedgehog", "X") + hmd.addKeyBinding("delete_hedgehog", "Y") + hmd.addKeyBinding("skip_to_next", "D") + hmd.addKeyBinding("skip_to_prev", "A") + hmd.addKeyBinding("faster_or_scale", "W") + hmd.addKeyBinding("slower_or_scale", "S") + hmd.addKeyBinding("play_pause", "M") + hmd.addKeyBinding("playback_direction", "N") + hmd.addKeyBinding("cell_division", "T") + + hmd.allowRepeats += OpenVRHMD.OpenVRButton.Trigger to TrackerRole.LeftHand + logger.info("calibration should start now") + + } + + fun launchHedgehogThread() { + thread { + while(!sciview.isInitialized) { Thread.sleep(200) } + + while(sciview.running) { + if(playing || skipToNext || skipToPrevious) { + val oldTimepoint = volume.viewerState.currentTimepoint + val newVolume = if(skipToNext || playing) { + skipToNext = false + if(direction == PlaybackDirection.Forward) { + volume.nextTimepoint() + } else { + volume.previousTimepoint() + } + } else { + skipToPrevious = false + if(direction == PlaybackDirection.Forward) { + volume.previousTimepoint() + } else { + volume.nextTimepoint() + } + } + val newTimepoint = volume.viewerState.currentTimepoint + + + if(hedgehogs.visible) { + if(hedgehogVisibility == HedgehogVisibility.PerTimePoint) { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { + if (it.metadata.isNotEmpty()) { + it.visible = (it.metadata["spine"] as SpineMetadata).timepoint == volume.viewerState.currentTimepoint + } + } + } + } else { + hedgehogs.children.forEach { hh -> + val hedgehog = hh as InstancedNode + hedgehog.instances.forEach { it.visible = true } + } + } + } + + if(tracking && oldTimepoint == (volume.timepointCount-1) && newTimepoint == 0) { + tracking = false + + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f)} + sciview.camera!!.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + dumpHedgehog() + } + } + + Thread.sleep((1000.0f/volumesPerSecond).toLong()) + } + } + } + + open fun addSpine(center: Vector3f, direction: Vector3f, volume: Volume, confidence: Float, timepoint: Int) { + val cam = sciview.camera as? DetachedHeadCamera ?: return + val sphere = volume.boundingBox?.getBoundingSphere() ?: return + + val sphereDirection = sphere.origin.minus(center) + val sphereDist = Math.sqrt(sphereDirection.x * sphereDirection.x + sphereDirection.y * sphereDirection.y + sphereDirection.z * sphereDirection.z) - sphere.radius + + val p1 = center + val temp = direction.mul(sphereDist + 2.0f * sphere.radius) + val p2 = Vector3f(center).add(temp) + + val spine = (hedgehogs.children.last() as InstancedNode).addInstance() + spine.spatial().orientBetweenPoints(p1, p2, true, true) + spine.visible = true + + val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) + if(intersection is MaybeIntersects.Intersection) { + // get local entry and exit coordinates, and convert to UV coords + val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) + val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) + // TODO: Allow for sampling a given time point of a volume +// val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) + // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now + val localDirection = Vector3f(0f) + val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) + val volumeScale = (volume as RAIVolume).getVoxelScale() +// if (samples != null && localDirection != null) { +// val metadata = SpineMetadata( +// timepoint, +// center, +// direction, +// intersection.distance, +// localEntry, +// localExit, +// localDirection, +// cam.headPosition, +// cam.headOrientation, +// cam.spatial().position, +// confidence, +// samples.map { it ?: 0.0f } +// ) + if (samples != null && samplePos != null) { + val metadata = SpineMetadata( + timepoint, + center, + direction, + intersection.distance, + localEntry, + localExit, + localDirection, + cam.headPosition, + cam.headOrientation, + cam.spatial().position, + confidence, + samples.map { it ?: 0.0f }, + samplePos.map { it?.mul(volumeScale) ?: Vector3f(0f) } + ) + val count = samples.filterNotNull().count { it > 0.2f } + + spine.metadata["spine"] = metadata + spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } + // TODO: Show confidence as color for the spine + spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + } + } + } + + /** + * Dumps a given hedgehog including created tracks to a file. + * If [hedgehog] is null, the last created hedgehog will be used, otherwise the given one. + * If [hedgehog] is not null, the cell track will not be added to the scene. + */ + fun dumpHedgehog(){ + logger.info("dumping hedgehog...") + val lastHedgehog = hedgehogs.children.last() as InstancedNode + val hedgehogId = hedgehogIds.incrementAndGet() + + val hedgehogFile = sessionDirectory.resolve("Hedgehog_${hedgehogId}_${SystemHelpers.formatDateTime()}.csv").toFile() + val hedgehogFileWriter = hedgehogFile.bufferedWriter() + hedgehogFileWriter.write("Timepoint,Origin,Direction,LocalEntry,LocalExit,LocalDirection,HeadPosition,HeadOrientation,Position,Confidence,Samples\n") + + val trackFile = sessionDirectory.resolve("Tracks.tsv").toFile() + val trackFileWriter = BufferedWriter(FileWriter(trackFile, true)) + if(!trackFile.exists()) { + trackFile.createNewFile() + trackFileWriter.write("# BionicTracking cell track listing for ${sessionDirectory.fileName}\n") + trackFileWriter.write("# TIME\tX\tYt\t\tZ\tTRACK_ID\tPARENT_TRACK_ID\tSPOT\tLABEL\n") + } + + + val spines = lastHedgehog.instances.mapNotNull { spine -> + spine.metadata["spine"] as? SpineMetadata + } + + spines.forEach { metadata -> + hedgehogFileWriter.write("${metadata.timepoint};${metadata.origin};${metadata.direction};${metadata.localEntry};${metadata.localExit};${metadata.localDirection};${metadata.headPosition};${metadata.headOrientation};${metadata.position};${metadata.confidence};${metadata.samples.joinToString(";")}\n") + } + hedgehogFileWriter.close() + + val existingAnalysis = lastHedgehog.metadata["HedgehogAnalysis"] as? HedgehogAnalysis.Track + val track = if(existingAnalysis is HedgehogAnalysis.Track) { + existingAnalysis + } else { + val h = HedgehogAnalysis(spines, Matrix4f(volume.spatial().world)) + h.run() + } + + if(track == null) { +// logger.warn("No track returned") + sciview.camera?.showMessage("No track returned", distance = 1.2f, size = 0.2f,messageColor = Vector4f(1.0f, 0.0f, 0.0f,1.0f)) + return + } + + lastHedgehog.metadata["HedgehogAnalysis"] = track + lastHedgehog.metadata["Spines"] = spines + +// logger.info("---\nTrack: ${track.points.joinToString("\n")}\n---") + +// val cylinder = Cylinder(0.1f, 1.0f, 6, smoothSides = true) +// cylinder.setMaterial(ShaderMaterial.fromFiles("DeferredInstancedColor.vert", "DeferredInstancedColor.frag")) { +// diffuse = Vector3f(1f) +// ambient = Vector3f(1f) +// roughness = 1f +// } + +// cylinder.name = "Track-$hedgehogId" +// val mainTrack = InstancedNode(cylinder) +// mainTrack.instancedProperties["Color"] = { Vector4f(1f) } + + val parentId = 0 + val volumeDimensions = volume.getDimensions() + + trackFileWriter.newLine() + trackFileWriter.newLine() + trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") + if (mastodonCallbackLinkCreate != null && mastodonUpdateGraph != null) { + track.points.windowed(2, 1).forEach { pair -> + mastodonCallbackLinkCreate?.let { it(pair[0].second) } +// val element = mainTrack.addInstance() +// element.addAttribute(Material::class.java, cylinder.material()) +// element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) +// element.parent = volume +// mainTrack.instances.add(element) + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + mastodonUpdateGraph?.invoke() + } else { + track.points.windowed(2, 1).forEach { pair -> + val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product + val tp = pair[0].second.timepoint + trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") + } + } + + +// mainTrack.let { sciview.addNode(it, parent = volume) } + + trackFileWriter.close() + } + +} \ No newline at end of file From 8f05cc47283da95509f8be543cf528ac65d046d9 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:40:45 +0200 Subject: [PATCH 46/55] Make sure that cam is not inside volume when tracking --- .../sc/iview/process/CellTrackingBase.kt | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/main/kotlin/sc/iview/process/CellTrackingBase.kt b/src/main/kotlin/sc/iview/process/CellTrackingBase.kt index 80378e1e..08ddb79b 100644 --- a/src/main/kotlin/sc/iview/process/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/process/CellTrackingBase.kt @@ -296,34 +296,23 @@ open class CellTrackingBase { val spine = (hedgehogs.children.last() as InstancedNode).addInstance() spine.spatial().orientBetweenPoints(p1, p2, true, true) - spine.visible = true + spine.visible = false val intersection = volume.spatial().intersectAABB(p1, (p2 - p1).normalize(), true) + + if (volume.boundingBox?.isInside(cam.spatial().position)!!) { + logger.info("Can't track inside the volume! Please move out of the volume and try again") + return + } if(intersection is MaybeIntersects.Intersection) { // get local entry and exit coordinates, and convert to UV coords - val localEntry = (intersection.relativeEntry) //.add(Vector3f(1.0f)) ) .mul (1.0f / 2.0f) - val localExit = (intersection.relativeExit) //.add (Vector3f(1.0f)) ).mul (1.0f / 2.0f) - // TODO: Allow for sampling a given time point of a volume -// val (samples, localDirection) = volume.sampleRay(localEntry, localExit) ?: (null to null) + val localEntry = (intersection.relativeEntry) + val localExit = (intersection.relativeExit) // TODO We dont need the local direction for grid traversal, but its still in the spine metadata for now val localDirection = Vector3f(0f) val (samples, samplePos) = volume.sampleRayGridTraversal(localEntry, localExit) ?: (null to null) val volumeScale = (volume as RAIVolume).getVoxelScale() -// if (samples != null && localDirection != null) { -// val metadata = SpineMetadata( -// timepoint, -// center, -// direction, -// intersection.distance, -// localEntry, -// localExit, -// localDirection, -// cam.headPosition, -// cam.headOrientation, -// cam.spatial().position, -// confidence, -// samples.map { it ?: 0.0f } -// ) + if (samples != null && samplePos != null) { val metadata = SpineMetadata( timepoint, @@ -345,7 +334,8 @@ open class CellTrackingBase { spine.metadata["spine"] = metadata spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } // TODO: Show confidence as color for the spine - spine.instancedProperties["Metadata"] = { Vector4f(confidence, timepoint.toFloat()/volume.timepointCount, count.toFloat(), 0.0f) } + spine.instancedProperties["Metadata"] = + { Vector4f(confidence, timepoint.toFloat() / volume.timepointCount, count.toFloat(), 0.0f) } } } } From 9bcafe8b42ea99b9eef6d33b1b896bfb4ed44bad Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:41:20 +0200 Subject: [PATCH 47/55] Add names to scene objects to make it easier to find and remove them again --- .../sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 5fa3c44e..65985b84 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -55,8 +55,6 @@ class EyeTrackingDemo: Command, CellTrackingBase() { @Parameter override var mastodonUpdateGraph: (() -> Unit)? = null - - val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) val calibrationTarget = Icosphere(0.02f, 2) @@ -80,6 +78,7 @@ class EyeTrackingDemo: Command, CellTrackingBase() { metallic = 0.0f diffuse = Vector3f(0.8f, 0.8f, 0.8f) } + referenceTarget.name = "Reference Target" sciview.camera?.addChild(referenceTarget) calibrationTarget.visible = false @@ -87,6 +86,7 @@ class EyeTrackingDemo: Command, CellTrackingBase() { roughness = 1.0f metallic = 0.0f diffuse = Vector3f(1.0f, 1.0f, 1.0f)} + calibrationTarget.name = "Calibration Target" sciview.camera?.addChild(calibrationTarget) laser.visible = false @@ -100,6 +100,7 @@ class EyeTrackingDemo: Command, CellTrackingBase() { diffuse = Vector3f(0.4f, 0.4f, 0.4f) } shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + shell.name = "Shell" sciview.addNode(shell) val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } @@ -239,7 +240,7 @@ class EyeTrackingDemo: Command, CellTrackingBase() { hmd.addKeyBinding("toggle_tracking", keybindingTracking) volume.visible = true - volume.runRecursive { it.visible = true } +// volume.runRecursive { it.visible = true } playing = true } From e6c86124743d32444ec366e1da3d813d294c5bf6 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:51:34 +0200 Subject: [PATCH 48/55] CellTrackingBase: moved file to demo/advanced --- .../demo/advanced}/CellTrackingBase.kt | 5 +---- .../sc/iview/commands/demo/advanced/EyeTrackingDemo.kt | 10 ---------- .../kotlin/sc/iview/commands/demo/advanced/Test.kt | 5 ----- .../commands/demo/advanced/VRControllerTrackingDemo.kt | 3 --- .../commands/demo/advanced/VRHeadSetTrackingDemo.kt | 2 -- 5 files changed, 1 insertion(+), 24 deletions(-) rename src/main/kotlin/sc/iview/{process => commands/demo/advanced}/CellTrackingBase.kt (98%) diff --git a/src/main/kotlin/sc/iview/process/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt similarity index 98% rename from src/main/kotlin/sc/iview/process/CellTrackingBase.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt index 08ddb79b..364d0cdd 100644 --- a/src/main/kotlin/sc/iview/process/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt @@ -1,4 +1,4 @@ -package sc.iview.process +package sc.iview.commands.demo.advanced import graphics.scenery.* import graphics.scenery.controls.OpenVRHMD @@ -18,9 +18,6 @@ import org.scijava.log.LogService import org.scijava.plugin.Parameter import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView -import sc.iview.commands.demo.advanced.ConfirmableClickBehaviour -import sc.iview.commands.demo.advanced.HedgehogAnalysis -import sc.iview.commands.demo.advanced.SpineMetadata import java.io.BufferedWriter import java.io.FileWriter import java.nio.file.Path diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index 65985b84..a55f6809 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -3,11 +3,8 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.TrackerRole -import graphics.scenery.controls.behaviours.ControllerDrag import graphics.scenery.controls.eyetracking.PupilEyeTracker import graphics.scenery.textures.Texture -import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz @@ -24,23 +21,16 @@ import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView import sc.iview.commands.MenuWeights import java.awt.image.DataBufferByte -import java.io.BufferedWriter import java.io.ByteArrayInputStream -import java.io.FileWriter import java.nio.file.Files -import java.nio.file.Path import java.nio.file.Paths import java.util.HashMap -import java.util.concurrent.atomic.AtomicInteger import javax.imageio.ImageIO import kotlin.concurrent.thread import kotlin.math.PI -import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import graphics.scenery.volumes.RAIVolume -import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt index 347a643d..45a33a3f 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt @@ -2,7 +2,6 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.OpenVRHMD import graphics.scenery.numerics.Random import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard @@ -16,17 +15,13 @@ import org.joml.* import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin import sc.iview.SciView import sc.iview.commands.MenuWeights -import sc.iview.process.CellTrackingBase import java.io.File import java.nio.file.Files -import java.nio.file.Path import java.nio.file.Paths import java.text.DecimalFormat -import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread @Plugin(type = Command::class, diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt index 241f1cf4..590e3c4f 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt @@ -10,7 +10,6 @@ import org.joml.* import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView @@ -19,12 +18,10 @@ import java.nio.file.Files import java.nio.file.Paths import java.util.HashMap import kotlin.concurrent.thread -import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.controls.* import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard -import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt index fd9bfab5..6e7bbe59 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt @@ -20,7 +20,6 @@ import java.nio.file.Files import java.nio.file.Paths import java.util.HashMap import kotlin.concurrent.thread -import org.scijava.log.LogService import graphics.scenery.attribute.material.Material import graphics.scenery.controls.behaviours.* import graphics.scenery.primitives.TextBoard @@ -28,7 +27,6 @@ import graphics.scenery.utils.extensions.* import org.scijava.event.EventService import sc.iview.commands.file.OpenDirofTif import sc.iview.event.NodeTaggedEvent -import sc.iview.process.CellTrackingBase @Plugin(type = Command::class, menuRoot = "SciView", From 886997a45c0222de65c918cc54c0ac35fba64c9e Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:03:53 +0200 Subject: [PATCH 49/55] Refactor the eye tracking logic into EyeTracking, so that EyeTrackingDemo is only a wrapper for command purposes --- .../commands/demo/advanced/EyeTracking.kt | 272 ++++++++++++++++++ .../commands/demo/advanced/EyeTrackingDemo.kt | 244 +--------------- 2 files changed, 281 insertions(+), 235 deletions(-) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt new file mode 100644 index 00000000..e1eef506 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt @@ -0,0 +1,272 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.controls.eyetracking.PupilEyeTracker +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard +import graphics.scenery.textures.Texture +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw +import graphics.scenery.volumes.Volume +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.* +import org.scijava.ui.behaviour.ClickBehaviour +import java.awt.image.DataBufferByte +import java.io.ByteArrayInputStream +import java.nio.file.Files +import java.nio.file.Paths +import javax.imageio.ImageIO +import kotlin.concurrent.thread +import kotlin.math.PI + +/** + * Tracking class used for communicating with eye trackers, tracking cells with them in a sciview VR environment. + * It calls the Hedgehog analysis on the eye tracking results and communicates the results to Mastodon via + * [mastodonCallbackLinkCreate], which is called on every spine graph vertex that is extracted, and + * [mastodonUpdateGraph] which is called after all vertices are iterated, giving Mastodon a chance to rebuild its tracks. + */ +class EyeTracking( + override var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null, + override var mastodonUpdateGraph: (() -> Unit)? = null +): CellTrackingBase() { + + val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) + + val calibrationTarget = Icosphere(0.02f, 2) + val laser = Cylinder(0.005f, 0.2f, 10) + + val confidenceThreshold = 0.60f + +// var currentVolume = 0 + + fun run() { + + sciview.toggleVRRendering() + logger.info("VR mode has been toggled") + hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") + sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" + sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) + + referenceTarget.visible = false + referenceTarget.ifMaterial{ + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(0.8f, 0.8f, 0.8f) + } + referenceTarget.name = "Reference Target" + sciview.camera?.addChild(referenceTarget) + + calibrationTarget.visible = false + calibrationTarget.material { + roughness = 1.0f + metallic = 0.0f + diffuse = Vector3f(1.0f, 1.0f, 1.0f) + } + calibrationTarget.name = "Calibration Target" + sciview.camera?.addChild(calibrationTarget) + + laser.visible = false + laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f) } + laser.name = "Laser" + sciview.addNode(laser) + + val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) + shell.ifMaterial{ + cullingMode = Material.CullingMode.Front + diffuse = Vector3f(0.4f, 0.4f, 0.4f) } + + shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) + shell.name = "Shell" + sciview.addNode(shell) + + val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } + + val v = (volnodes.firstOrNull() as? Volume) + if(v == null) { + logger.warn("No volume found, bailing") + return + } else { + logger.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") + volume = v + } + volume.visible = false + + val bb = BoundingGrid() + bb.node = volume + bb.visible = false + + sciview.addNode(hedgehogs) + + val eyeFrames = Mesh("eyeFrames") + val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) + val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) + left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) + left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) + right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) + eyeFrames.addChild(left) + eyeFrames.addChild(right) + + sciview.addNode(eyeFrames) + + val pupilFrameLimit = 20 + var lastFrame = System.nanoTime() + + pupilTracker.subscribeFrames { eye, texture -> + if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { + return@subscribeFrames + } + + val node = if(eye == 1) { + left + } else { + right + } + + val stream = ByteArrayInputStream(texture) + val image = ImageIO.read(stream) + val data = (image.raster.dataBuffer as DataBufferByte).data + + node.ifMaterial { + textures["diffuse"] = Texture( + Vector3i(image.width, image.height, 1), + 3, + UnsignedByteType(), + BufferUtils.allocateByteAndPut(data) + ) } + + lastFrame = System.nanoTime() + } + + // TODO: Replace with cam.showMessage() + val debugBoard = TextBoard() + debugBoard.name = "debugBoard" + debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.text = "" + debugBoard.visible = false + sciview.camera?.addChild(debugBoard) + + val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) + lights.forEach { sciview.addNode(it) } + + thread { + logger.info("Adding onDeviceConnect handlers") + hmd.events.onDeviceConnect.add { hmd, device, timestamp -> + logger.info("onDeviceConnect called, cam=${sciview.camera}") + if(device.type == TrackedDeviceType.Controller) { + logger.info("Got device ${device.name} at $timestamp") + device.model?.let { hmd.attachToNode(device, it, sciview.camera) } + } + } + } + thread{ + logger.info("started thread for inputSetup") + inputSetup() + setupCalibration() + } + + launchHedgehogThread() + } + + + private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { + val startCalibration = ClickBehaviour { _, _ -> + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + pupilTracker.gazeConfidenceThreshold = confidenceThreshold + if (!pupilTracker.isCalibrated) { + logger.info("pupil is currently uncalibrated") + pupilTracker.onCalibrationInProgress = { + cam.showMessage("Crunching equations ...",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000, centered = true) + } + + pupilTracker.onCalibrationFailed = { + cam.showMessage("Calibration failed.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), centered = true) + } + + pupilTracker.onCalibrationSuccess = { + cam.showMessage("Calibration succeeded!", distance = 2f, size = 0.2f, messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), centered = true) +// cam.children.find { it.name == "debugBoard" }?.visible = true + + for (i in 0 until 20) { + referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f) } + Thread.sleep(100) + referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f) } + Thread.sleep(30) + } + + hmd.removeBehaviour("start_calibration") + hmd.removeKeyBinding("start_calibration") + + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + logger.info("deactivating tracking...") + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) + dumpHedgehog() + } else { + logger.info("activating tracking...") + addHedgehog() + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) + } + tracking = !tracking + } + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) + + volume.visible = true +// volume.runRecursive { it.visible = true } + playing = true + } + + pupilTracker.unsubscribeFrames() + sciview.deleteNode(sciview.find("eyeFrames")) + + logger.info("Starting eye tracker calibration") + cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) + pupilTracker.calibrate(cam, hmd, + generateReferenceData = true, + calibrationTarget = calibrationTarget) + + pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { + //NEW + PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> + if (gaze.confidence > confidenceThreshold) { + val p = gaze.gazePoint() + referenceTarget.visible = true + // Pupil has mm units, so we divide by 1000 here to get to scenery units + referenceTarget.spatial().position = p + (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" + + val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) + val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() + val direction = (pointWorld - headCenter).normalize() + + if (tracking) { +// log.info("Starting spine from $headCenter to $pointWorld") + addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) + } + } + } + +// else -> {gaze-> } + } + + logger.info("Calibration routine done.") + } + + // bind calibration start to menu key on controller + + } + } + hmd.addBehaviour("start_calibration", startCalibration) + hmd.addKeyBinding("start_calibration", keybindingCalibration) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index a55f6809..cf00b017 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -37,249 +37,23 @@ import graphics.scenery.primitives.TextBoard menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class EyeTrackingDemo: Command, CellTrackingBase() { +/** + * Command class that forwards to the [EyeTracking] class to perform the actual tracking and analysis. + */ +class EyeTrackingDemo: Command { @Parameter - override var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null + var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null @Parameter - override var mastodonUpdateGraph: (() -> Unit)? = null - - val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) - - val calibrationTarget = Icosphere(0.02f, 2) - val laser = Cylinder(0.005f, 0.2f, 10) - - val confidenceThreshold = 0.60f - -// var currentVolume = 0 + var mastodonUpdateGraph: (() -> Unit)? = null override fun run() { - - sciview.toggleVRRendering() - logger.info("VR mode has been toggled") - hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") - sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" - sessionDirectory = Files.createDirectory(Paths.get(System.getProperty("user.home"), "Desktop", sessionId)) - - referenceTarget.visible = false - referenceTarget.ifMaterial{ - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(0.8f, 0.8f, 0.8f) - } - referenceTarget.name = "Reference Target" - sciview.camera?.addChild(referenceTarget) - - calibrationTarget.visible = false - calibrationTarget.material { - roughness = 1.0f - metallic = 0.0f - diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - calibrationTarget.name = "Calibration Target" - sciview.camera?.addChild(calibrationTarget) - - laser.visible = false - laser.ifMaterial{diffuse = Vector3f(1.0f, 1.0f, 1.0f)} - laser.name = "Laser" - sciview.addNode(laser) - - val shell = Box(Vector3f(20.0f, 20.0f, 20.0f), insideNormals = true) - shell.ifMaterial{ - cullingMode = Material.CullingMode.Front - diffuse = Vector3f(0.4f, 0.4f, 0.4f) } - - shell.spatial().position = Vector3f(0.0f, 0.0f, 0.0f) - shell.name = "Shell" - sciview.addNode(shell) - - val volnodes = sciview.findNodes { node -> Volume::class.java.isAssignableFrom(node.javaClass) } - - val v = (volnodes.firstOrNull() as? Volume) - if(v == null) { - logger.warn("No volume found, bailing") - return - } else { - logger.info("found ${volnodes.size} volume nodes. Using the first one: ${volnodes.first()}") - volume = v - } - volume.visible = false - - val bb = BoundingGrid() - bb.node = volume - bb.visible = false - - sciview.addNode(hedgehogs) - - val eyeFrames = Mesh("eyeFrames") - val left = Box(Vector3f(1.0f, 1.0f, 0.001f)) - val right = Box(Vector3f(1.0f, 1.0f, 0.001f)) - left.spatial().position = Vector3f(-1.0f, 1.5f, 0.0f) - left.spatial().rotation = left.spatial().rotation.rotationZ(PI.toFloat()) - right.spatial().position = Vector3f(1.0f, 1.5f, 0.0f) - eyeFrames.addChild(left) - eyeFrames.addChild(right) - - sciview.addNode(eyeFrames) - - val pupilFrameLimit = 20 - var lastFrame = System.nanoTime() - - pupilTracker.subscribeFrames { eye, texture -> - if(System.nanoTime() - lastFrame < pupilFrameLimit*10e5) { - return@subscribeFrames - } - - val node = if(eye == 1) { - left - } else { - right - } - - val stream = ByteArrayInputStream(texture) - val image = ImageIO.read(stream) - val data = (image.raster.dataBuffer as DataBufferByte).data - - node.ifMaterial { - textures["diffuse"] = Texture( - Vector3i(image.width, image.height, 1), - 3, - UnsignedByteType(), - BufferUtils.allocateByteAndPut(data) - ) } - - lastFrame = System.nanoTime() - } - - // TODO: Replace with cam.showMessage() - val debugBoard = TextBoard() - debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) - debugBoard.text = "" - debugBoard.visible = false - sciview.camera?.addChild(debugBoard) - - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addNode(it) } - - thread { - logger.info("Adding onDeviceConnect handlers") - hmd.events.onDeviceConnect.add { hmd, device, timestamp -> - logger.info("onDeviceConnect called, cam=${sciview.camera}") - if(device.type == TrackedDeviceType.Controller) { - logger.info("Got device ${device.name} at $timestamp") - device.model?.let { hmd.attachToNode(device, it, sciview.camera) } - } - } - } - thread{ - logger.info("started thread for inputSetup") - inputSetup() - setupCalibration() - } - - launchHedgehogThread() - } - - - private fun setupCalibration(keybindingCalibration: String = "N", keybindingTracking: String = "U") { - val startCalibration = ClickBehaviour { _, _ -> - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - pupilTracker.gazeConfidenceThreshold = confidenceThreshold - if (!pupilTracker.isCalibrated) { - logger.info("pupil is currently uncalibrated") - pupilTracker.onCalibrationInProgress = { - cam.showMessage("Crunching equations ...",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.8f, 0.0f, 1.0f), duration = 15000, centered = true) - } - - pupilTracker.onCalibrationFailed = { - cam.showMessage("Calibration failed.",distance = 2f, size = 0.2f, messageColor = Vector4f(1.0f, 0.0f, 0.0f, 1.0f), centered = true) - } - - pupilTracker.onCalibrationSuccess = { - cam.showMessage("Calibration succeeded!", distance = 2f, size = 0.2f, messageColor = Vector4f(0.0f, 1.0f, 0.0f, 1.0f), centered = true) -// cam.children.find { it.name == "debugBoard" }?.visible = true - - for (i in 0 until 20) { - referenceTarget.ifMaterial{diffuse = Vector3f(0.0f, 1.0f, 0.0f)} - Thread.sleep(100) - referenceTarget.ifMaterial { diffuse = Vector3f(0.8f, 0.8f, 0.8f)} - Thread.sleep(30) - } - - hmd.removeBehaviour("start_calibration") - hmd.removeKeyBinding("start_calibration") - - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - logger.info("deactivating tracking...") - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 2f, size = 0.2f, centered = true) - dumpHedgehog() - } else { - logger.info("activating tracking...") - addHedgehog() - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 2f, size = 0.2f, centered = true) - } - tracking = !tracking - } - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) - - volume.visible = true -// volume.runRecursive { it.visible = true } - playing = true - } - - pupilTracker.unsubscribeFrames() - sciview.deleteNode(sciview.find("eyeFrames")) - - logger.info("Starting eye tracker calibration") - cam.showMessage("Follow the white rabbit.", distance = 2f, size = 0.2f,duration = 1500, centered = true) - pupilTracker.calibrate(cam, hmd, - generateReferenceData = true, - calibrationTarget = calibrationTarget) - - pupilTracker.onGazeReceived = when (pupilTracker.calibrationType) { - //NEW - PupilEyeTracker.CalibrationType.WorldSpace -> { gaze -> - if (gaze.confidence > confidenceThreshold) { - val p = gaze.gazePoint() - referenceTarget.visible = true - // Pupil has mm units, so we divide by 1000 here to get to scenery units - referenceTarget.spatial().position = p - (cam.children.find { it.name == "debugBoard" } as? TextBoard)?.text = "${String.format("%.2f", p.x())}, ${String.format("%.2f", p.y())}, ${String.format("%.2f", p.z())}" - - val headCenter = cam.spatial().viewportToWorld(Vector2f(0.0f, 0.0f)) - val pointWorld = Matrix4f(cam.spatial().world).transform(p.xyzw()).xyz() - val direction = (pointWorld - headCenter).normalize() - - if (tracking) { -// log.info("Starting spine from $headCenter to $pointWorld") - addSpine(headCenter, direction, volume, gaze.confidence, volume.viewerState.currentTimepoint) - } - } - } - -// else -> {gaze-> } - } - - logger.info("Calibration routine done.") - } - - // bind calibration start to menu key on controller - - } - } - hmd.addBehaviour("start_calibration", startCalibration) - hmd.addKeyBinding("start_calibration", keybindingCalibration) + // the actual eye tracking logic happens in here + val eyeTracking = EyeTracking(mastodonCallbackLinkCreate, mastodonUpdateGraph) + eyeTracking.run() } - - companion object { @Throws(Exception::class) From 6d93dc073836f2d90d649a57ac77c64775d8ba20 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:14:57 +0200 Subject: [PATCH 50/55] Add proper eyetracking shutdown handling --- .../demo/advanced/CellTrackingBase.kt | 37 +++++++++----- .../commands/demo/advanced/EyeTracking.kt | 51 ++++++++++++++----- .../commands/demo/advanced/EyeTrackingDemo.kt | 6 ++- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt index 364d0cdd..93871111 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt @@ -24,12 +24,9 @@ import java.nio.file.Path import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread -open class CellTrackingBase { - - @Parameter - lateinit var sciview: SciView - - @Parameter +open class CellTrackingBase( + open var sciview: SciView +) { lateinit var logger: LogService lateinit var sessionId: String @@ -52,8 +49,10 @@ open class CellTrackingBase { var volumeScaleFactor = 1.0f - open var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null - open var mastodonUpdateGraph: (() -> Unit)? = null + var cellTrackingActive: Boolean = false + + open var linkCreationCallback: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null + open var finalTrackCallback: (() -> Unit)? = null enum class HedgehogVisibility { Hidden, PerTimePoint, Visible } var hedgehogVisibility = HedgehogVisibility.Hidden @@ -223,11 +222,14 @@ open class CellTrackingBase { } - fun launchHedgehogThread() { + /** + * Launches a thread that updates the volume time points, the hedgehog visibility and reference target color. + */ + fun launchUpdaterThread() { thread { while(!sciview.isInitialized) { Thread.sleep(200) } - while(sciview.running) { + while(sciview.running && cellTrackingActive) { if(playing || skipToNext || skipToPrevious) { val oldTimepoint = volume.viewerState.currentTimepoint val newVolume = if(skipToNext || playing) { @@ -405,9 +407,9 @@ open class CellTrackingBase { trackFileWriter.newLine() trackFileWriter.newLine() trackFileWriter.write("# START OF TRACK $hedgehogId, child of $parentId\n") - if (mastodonCallbackLinkCreate != null && mastodonUpdateGraph != null) { + if (linkCreationCallback != null && finalTrackCallback != null) { track.points.windowed(2, 1).forEach { pair -> - mastodonCallbackLinkCreate?.let { it(pair[0].second) } + linkCreationCallback?.let { it(pair[0].second) } // val element = mainTrack.addInstance() // element.addAttribute(Material::class.java, cylinder.material()) // element.spatial().orientBetweenPoints(Vector3f(pair[0].first), Vector3f(pair[1].first), rescale = true, reposition = true) @@ -417,7 +419,7 @@ open class CellTrackingBase { val tp = pair[0].second.timepoint trackFileWriter.write("$tp\t${p.x()}\t${p.y()}\t${p.z()}\t${hedgehogId}\t$parentId\t0\t0\n") } - mastodonUpdateGraph?.invoke() + finalTrackCallback?.invoke() } else { track.points.windowed(2, 1).forEach { pair -> val p = Vector3f(pair[0].first).mul(Vector3f(volumeDimensions)) // direct product @@ -432,4 +434,13 @@ open class CellTrackingBase { trackFileWriter.close() } + /** + * Stops the current tracking environment and restore the original state. + * This method should be overridden to extend + */ + open fun stop() { + hmd.close() + logger.info("Shut down HMD and keybindings.") + } + } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt index e1eef506..7e4d0716 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt @@ -16,6 +16,7 @@ import graphics.scenery.volumes.Volume import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* import org.scijava.ui.behaviour.ClickBehaviour +import sc.iview.SciView import java.awt.image.DataBufferByte import java.io.ByteArrayInputStream import java.nio.file.Files @@ -27,26 +28,29 @@ import kotlin.math.PI /** * Tracking class used for communicating with eye trackers, tracking cells with them in a sciview VR environment. * It calls the Hedgehog analysis on the eye tracking results and communicates the results to Mastodon via - * [mastodonCallbackLinkCreate], which is called on every spine graph vertex that is extracted, and - * [mastodonUpdateGraph] which is called after all vertices are iterated, giving Mastodon a chance to rebuild its tracks. + * [linkCreationCallback], which is called on every spine graph vertex that is extracted, and + * [finalTrackCallback] which is called after all vertices of a track are iterated, giving Mastodon a chance to rebuild its tracks. */ class EyeTracking( - override var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null, - override var mastodonUpdateGraph: (() -> Unit)? = null -): CellTrackingBase() { + override var linkCreationCallback: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null, + override var finalTrackCallback: (() -> Unit)? = null, + sciview: SciView +): CellTrackingBase(sciview) { val pupilTracker = PupilEyeTracker(calibrationType = PupilEyeTracker.CalibrationType.WorldSpace, port = System.getProperty("PupilPort", "50020").toInt()) - val calibrationTarget = Icosphere(0.02f, 2) val laser = Cylinder(0.005f, 0.2f, 10) val confidenceThreshold = 0.60f - // var currentVolume = 0 + private lateinit var lightTetrahedron: List + private lateinit var debugBoard: TextBoard + fun run() { sciview.toggleVRRendering() + cellTrackingActive = true logger.info("VR mode has been toggled") hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") sessionId = "BionicTracking-generated-${SystemHelpers.formatDateTime()}" @@ -143,16 +147,21 @@ class EyeTracking( } // TODO: Replace with cam.showMessage() - val debugBoard = TextBoard() + debugBoard = TextBoard() debugBoard.name = "debugBoard" - debugBoard.scale = Vector3f(0.05f, 0.05f, 0.05f) - debugBoard.position = Vector3f(0.0f, -0.3f, -0.9f) + debugBoard.spatial().scale = Vector3f(0.05f, 0.05f, 0.05f) + debugBoard.spatial().position = Vector3f(0.0f, -0.3f, -0.9f) debugBoard.text = "" debugBoard.visible = false sciview.camera?.addChild(debugBoard) - val lights = Light.createLightTetrahedron(Vector3f(0.0f, 0.0f, 0.0f), spread = 5.0f, radius = 15.0f, intensity = 5.0f) - lights.forEach { sciview.addNode(it) } + lightTetrahedron = Light.createLightTetrahedron( + Vector3f(0.0f, 0.0f, 0.0f), + spread = 5.0f, + radius = 15.0f, + intensity = 5.0f + ) + lightTetrahedron.forEach { sciview.addNode(it) } thread { logger.info("Adding onDeviceConnect handlers") @@ -170,7 +179,7 @@ class EyeTracking( setupCalibration() } - launchHedgehogThread() + launchUpdaterThread() } @@ -269,4 +278,20 @@ class EyeTracking( hmd.addKeyBinding("start_calibration", keybindingCalibration) } + override fun stop() { + logger.info("Shutting down eye tracking environment. Disabling VR now...") + sciview.toggleVRRendering() + logger.info("Stopping volume and hedgehog updater thread...") + cellTrackingActive = false + logger.info("Deleting eye tracking scene objects...") + lightTetrahedron.forEach { sciview.deleteNode(it) } + sciview.deleteNode(sciview.find("Shell")) + listOf(referenceTarget, calibrationTarget, laser, debugBoard, hedgehogs).forEach { + sciview.deleteNode(it) + } + logger.info("Shutting down HMD and keybindings...") + hmd.close() + logger.info("Successfully cleaned up eye tracking environemt.") + } + } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt index cf00b017..06029046 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt @@ -48,9 +48,12 @@ class EyeTrackingDemo: Command { @Parameter var mastodonUpdateGraph: (() -> Unit)? = null + @Parameter + private lateinit var sv: SciView + override fun run() { // the actual eye tracking logic happens in here - val eyeTracking = EyeTracking(mastodonCallbackLinkCreate, mastodonUpdateGraph) + val eyeTracking = EyeTracking(mastodonCallbackLinkCreate, mastodonUpdateGraph, sv) eyeTracking.run() } @@ -62,6 +65,7 @@ class EyeTrackingDemo: Command { val sv = SciView.create() val command = sv.scijavaContext!!.getService(CommandService::class.java) val argmap = HashMap() + argmap["sv"] = sv command.run(EyeTrackingDemo::class.java, true, argmap) } } From 9cafe6472dae87813b0c586cf877f29a97a9fcc2 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:12:54 +0200 Subject: [PATCH 51/55] Unified naming scheme, extracted tracking functionality into separate classes --- .../demo/advanced/EyeTrackingCommand.kt | 53 +++++++ .../advanced/{Test.kt => TrackingTest.kt} | 33 ++-- .../demo/advanced/TrackingTestCommand.kt | 39 +++++ ...rackingDemo.kt => VRControllerTracking.kt} | 53 +++---- ...Demo.kt => VRControllerTrackingCommand.kt} | 43 ++---- .../demo/advanced/VRHeadSetTrackingCommand.kt | 60 ++++++++ ...etTrackingDemo.kt => VRHeadsetTracking.kt} | 142 +++++++----------- 7 files changed, 249 insertions(+), 174 deletions(-) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingCommand.kt rename src/main/kotlin/sc/iview/commands/demo/advanced/{Test.kt => TrackingTest.kt} (92%) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTestCommand.kt rename src/main/kotlin/sc/iview/commands/demo/advanced/{VRControllerTrackingDemo.kt => VRControllerTracking.kt} (87%) rename src/main/kotlin/sc/iview/commands/demo/advanced/{EyeTrackingDemo.kt => VRControllerTrackingCommand.kt} (50%) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingCommand.kt rename src/main/kotlin/sc/iview/commands/demo/advanced/{VRHeadSetTrackingDemo.kt => VRHeadsetTracking.kt} (54%) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingCommand.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingCommand.kt new file mode 100644 index 00000000..44298aec --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingCommand.kt @@ -0,0 +1,53 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.util.HashMap + +@Plugin( + type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)] +) +/** + * Command class that forwards to the [EyeTracking] class to perform the actual tracking and analysis. + */ +class EyeTrackingCommand : Command { + + @Parameter + var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null + + @Parameter + var mastodonUpdateGraph: (() -> Unit)? = null + + @Parameter + private lateinit var sv: SciView + + override fun run() { + // the actual eye tracking logic happens in here + val eyeTracking = EyeTracking(mastodonCallbackLinkCreate, mastodonUpdateGraph, sv) + eyeTracking.run() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + argmap["sv"] = sv + command.run(EyeTrackingCommand::class.java, true, argmap) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt similarity index 92% rename from src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt index 45a33a3f..0fc0a445 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/Test.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt @@ -12,24 +12,21 @@ import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw import graphics.scenery.volumes.Volume import org.joml.* -import org.scijava.command.Command import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Plugin import sc.iview.SciView -import sc.iview.commands.MenuWeights import java.io.File import java.nio.file.Files import java.nio.file.Paths import java.text.DecimalFormat import kotlin.concurrent.thread -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class Test: Command, CellTrackingBase() { +/** + * A class to test to show tracks and perform track analysis from saved CSV tracking files without + * the requirement of a VR headset. + */ +class TrackingTest( + sciview: SciView +): CellTrackingBase(sciview) { //val calibrationTarget = Icosphere(0.02f, 2) val TestTarget = Icosphere(0.1f, 2) @@ -43,7 +40,7 @@ class Test: Command, CellTrackingBase() { // var currentVolume = 0 - override fun run() { + fun run() { sciview.addNode(TestTarget) TestTarget.visible = false @@ -117,7 +114,7 @@ class Test: Command, CellTrackingBase() { inputSetup() } - launchHedgehogThread() + launchUpdaterThread() } override fun inputSetup() @@ -295,16 +292,4 @@ class Test: Command, CellTrackingBase() { } } } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) - } - } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTestCommand.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTestCommand.kt new file mode 100644 index 00000000..9f97bcf8 --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTestCommand.kt @@ -0,0 +1,39 @@ +package sc.iview.commands.demo.advanced + +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights + +@Plugin(type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Test without VR and Eye Tracker", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) +class TrackingTestCommand: Command { + + @Parameter + private lateinit var sv: SciView + + override fun run() { + val test = TrackingTest(sv) + test.run() + } + + companion object { + + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + val argmap = HashMap() + argmap["sv"] = sv + command.run(TrackingTestCommand::class.java, true, argmap) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTracking.kt similarity index 87% rename from src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTracking.kt index 590e3c4f..31e5d675 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTracking.kt @@ -1,34 +1,32 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.controls.TrackedDevice +import graphics.scenery.controls.TrackedDeviceType +import graphics.scenery.primitives.Cylinder +import graphics.scenery.primitives.TextBoard import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw import graphics.scenery.volumes.Volume -import org.joml.* -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Plugin +import org.joml.Matrix4f +import org.joml.Vector3f import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView -import sc.iview.commands.MenuWeights import java.nio.file.Files import java.nio.file.Paths -import java.util.HashMap import kotlin.concurrent.thread -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.* -import graphics.scenery.primitives.Cylinder -import graphics.scenery.primitives.TextBoard -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRControllerTrackingDemo: Command, CellTrackingBase() { + +/** + * This class utilizes VR controllers to track cells in volumetric datasets in a sciview environment. + */ +class VRControllerTracking( + sciview: SciView +): CellTrackingBase(sciview) { val testTarget1 = Icosphere(0.01f, 2) val testTarget2 = Icosphere(0.04f, 2) @@ -40,7 +38,7 @@ class VRControllerTrackingDemo: Command, CellTrackingBase() { // var currentVolume = 0 - override fun run() { + fun run() { sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -126,7 +124,7 @@ class VRControllerTrackingDemo: Command, CellTrackingBase() { setupControllerforTracking() } - launchHedgehogThread() + launchUpdaterThread() } private fun setupControllerforTracking( keybindingTracking: String = "U") { @@ -196,25 +194,8 @@ class VRControllerTrackingDemo: Command, CellTrackingBase() { addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) } } - } - } - - } - - - } - - companion object { - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) } } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingCommand.kt similarity index 50% rename from src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingCommand.kt index 06029046..01a36c4f 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRControllerTrackingCommand.kt @@ -1,64 +1,47 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* -import graphics.scenery.controls.OpenVRHMD -import graphics.scenery.controls.TrackedDeviceType -import graphics.scenery.controls.eyetracking.PupilEyeTracker -import graphics.scenery.textures.Texture import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.xyz import graphics.scenery.utils.extensions.xyzw import graphics.scenery.volumes.Volume -import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.* import org.scijava.command.Command import org.scijava.command.CommandService import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter import org.scijava.plugin.Plugin import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView import sc.iview.commands.MenuWeights -import java.awt.image.DataBufferByte -import java.io.ByteArrayInputStream import java.nio.file.Files import java.nio.file.Paths import java.util.HashMap -import javax.imageio.ImageIO import kotlin.concurrent.thread -import kotlin.math.PI import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.* import graphics.scenery.primitives.Cylinder import graphics.scenery.primitives.TextBoard +import org.scijava.plugin.Parameter -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize Eye Tracker for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -/** - * Command class that forwards to the [EyeTracking] class to perform the actual tracking and analysis. - */ -class EyeTrackingDemo: Command { - - @Parameter - var mastodonCallbackLinkCreate: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null - - @Parameter - var mastodonUpdateGraph: (() -> Unit)? = null +@Plugin( + type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Controller for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)] +) +class VRControllerTrackingCommand : Command { @Parameter private lateinit var sv: SciView override fun run() { - // the actual eye tracking logic happens in here - val eyeTracking = EyeTracking(mastodonCallbackLinkCreate, mastodonUpdateGraph, sv) - eyeTracking.run() + val tracking = VRControllerTracking(sv) + tracking.run() } companion object { - @Throws(Exception::class) @JvmStatic fun main(args: Array) { @@ -66,7 +49,7 @@ class EyeTrackingDemo: Command { val command = sv.scijavaContext!!.getService(CommandService::class.java) val argmap = HashMap() argmap["sv"] = sv - command.run(EyeTrackingDemo::class.java, true, argmap) + command.run(VRControllerTrackingCommand::class.java, true, argmap) } } } \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingCommand.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingCommand.kt new file mode 100644 index 00000000..7ef9163e --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingCommand.kt @@ -0,0 +1,60 @@ +package sc.iview.commands.demo.advanced + +import graphics.scenery.* +import org.joml.* +import org.scijava.command.Command +import org.scijava.command.CommandService +import org.scijava.plugin.Menu +import org.scijava.plugin.Parameter +import org.scijava.plugin.Plugin +import sc.iview.SciView +import sc.iview.commands.MenuWeights +import java.io.File +import java.util.HashMap +import graphics.scenery.controls.behaviours.* +import graphics.scenery.utils.extensions.* +import org.scijava.event.EventService +import sc.iview.commands.file.OpenDirofTif + +@Plugin( + type = Command::class, + menuRoot = "SciView", + menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), + Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), + Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)] +) +class VRHeadSetTrackingCommand : Command { + + @Parameter + private lateinit var eventService: EventService + + @Parameter + private lateinit var sv: SciView + + override fun run() { + val tracking = VRHeadsetTracking(sv, eventService) + tracking.run() + } + + companion object { + //run function from here, it will automatically choose the volume for rendering, please give the correct location of volume + @Throws(Exception::class) + @JvmStatic + fun main(args: Array) { + val sv = SciView.create() + val command = sv.scijavaContext!!.getService(CommandService::class.java) + // TODO this should probably open a file open dialog instead of hardcoding a path? + command.run(OpenDirofTif::class.java, true, + hashMapOf( + "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), + "onlyFirst" to 10 + )) + .get() + + val argmap = HashMap() + argmap["sv"] = sv + command.run(VRHeadSetTrackingCommand::class.java, true, argmap) + .get() + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadsetTracking.kt similarity index 54% rename from src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt rename to src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadsetTracking.kt index 6e7bbe59..09866186 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadSetTrackingDemo.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/VRHeadsetTracking.kt @@ -1,42 +1,36 @@ package sc.iview.commands.demo.advanced import graphics.scenery.* +import graphics.scenery.attribute.material.Material import graphics.scenery.controls.OpenVRHMD import graphics.scenery.controls.TrackedDeviceType import graphics.scenery.controls.TrackerRole +import graphics.scenery.controls.behaviours.VRGrab +import graphics.scenery.controls.behaviours.VRSelect +import graphics.scenery.controls.behaviours.VRTouch +import graphics.scenery.primitives.TextBoard import graphics.scenery.utils.SystemHelpers +import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.xyzw import graphics.scenery.volumes.Volume -import org.joml.* -import org.scijava.command.Command -import org.scijava.command.CommandService -import org.scijava.plugin.Menu -import org.scijava.plugin.Parameter -import org.scijava.plugin.Plugin +import org.joml.Matrix4f +import org.joml.Vector3f +import org.scijava.event.EventService import org.scijava.ui.behaviour.ClickBehaviour import sc.iview.SciView -import sc.iview.commands.MenuWeights -import java.io.File +import sc.iview.event.NodeTaggedEvent import java.nio.file.Files import java.nio.file.Paths -import java.util.HashMap import kotlin.concurrent.thread -import graphics.scenery.attribute.material.Material -import graphics.scenery.controls.behaviours.* -import graphics.scenery.primitives.TextBoard -import graphics.scenery.utils.extensions.* -import org.scijava.event.EventService -import sc.iview.commands.file.OpenDirofTif -import sc.iview.event.NodeTaggedEvent - -@Plugin(type = Command::class, - menuRoot = "SciView", - menu = [Menu(label = "Demo", weight = MenuWeights.DEMO), - Menu(label = "Advanced", weight = MenuWeights.DEMO_ADVANCED), - Menu(label = "Utilize VR Headset for Cell Tracking", weight = MenuWeights.DEMO_ADVANCED_EYETRACKING)]) -class VRHeadSetTrackingDemo: Command, CellTrackingBase() { - @Parameter - private lateinit var eventService: EventService +/** + * This class uses the VR headset's orientation to track cells in volumetric datasets in a sciview environment. + */ +class VRHeadsetTracking( + sciview: SciView, + val eventService: EventService, +): CellTrackingBase(sciview) { var hedgehogsList = mutableListOf() @@ -44,7 +38,7 @@ class VRHeadSetTrackingDemo: Command, CellTrackingBase() { // var currentVolume = 0 - override fun run() { + fun run() { sciview.toggleVRRendering() hmd = sciview.hub.getWorkingHMD() as? OpenVRHMD ?: throw IllegalStateException("Could not find headset") @@ -105,7 +99,7 @@ class VRHeadSetTrackingDemo: Command, CellTrackingBase() { setupHeadsetTracking() } - launchHedgehogThread() + launchUpdaterThread() } private fun setupHeadsetTracking() { @@ -134,79 +128,59 @@ class VRHeadSetTrackingDemo: Command, CellTrackingBase() { true) - VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand,TrackerRole.RightHand),true) + VRTouch.createAndSet(sciview.currentScene,hmd, listOf(TrackerRole.LeftHand, TrackerRole.RightHand),true) - VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf(TrackerRole.RightHand,TrackerRole.LeftHand)) + VRGrab.createAndSet(sciview.currentScene,hmd, listOf(OpenVRHMD.OpenVRButton.Side), listOf( + TrackerRole.RightHand, + TrackerRole.LeftHand)) setupControllerforTracking() } private fun setupControllerforTracking( keybindingTracking: String = "U") { - thread { - val cam = sciview.camera as? DetachedHeadCamera ?: return@thread - val toggleTracking = ClickBehaviour { _, _ -> - if (tracking) { - referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } - cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) - tracking = false - thread { - dumpHedgehog() - println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) - } - } else { - addHedgehog() - println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) - referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } - cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) - tracking = true + thread { + val cam = sciview.camera as? DetachedHeadCamera ?: return@thread + val toggleTracking = ClickBehaviour { _, _ -> + if (tracking) { + referenceTarget.ifMaterial { diffuse = Vector3f(0.5f, 0.5f, 0.5f) } + cam.showMessage("Tracking deactivated.",distance = 1.2f, size = 0.2f) + tracking = false + thread { + dumpHedgehog() + println("before dumphedgehog: " + hedgehogsList.last().instances.size.toString()) } + } else { + addHedgehog() + println("after addhedgehog: "+ hedgehogsList.last().instances.size.toString()) + referenceTarget.ifMaterial { diffuse = Vector3f(1.0f, 0.0f, 0.0f) } + cam.showMessage("Tracking active.",distance = 1.2f, size = 0.2f) + tracking = true } - //RightController.trigger - hmd.addBehaviour("toggle_tracking", toggleTracking) - hmd.addKeyBinding("toggle_tracking", keybindingTracking) + } + //RightController.trigger + hmd.addBehaviour("toggle_tracking", toggleTracking) + hmd.addKeyBinding("toggle_tracking", keybindingTracking) - volume.visible = true - volume.runRecursive { it.visible = true } + volume.visible = true + volume.runRecursive { it.visible = true } // playing = false - while(true) - { + while(true) + { - val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() - val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() + val headCenter = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-1f).xyzw()).xyz() + val pointWorld = Matrix4f(cam.spatial().world).transform(Vector3f(0.0f,0f,-2f).xyzw()).xyz() - referenceTarget.visible = true - referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } - - val direction = (pointWorld - headCenter).normalize() - if (tracking) { - addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) - } + referenceTarget.visible = true + referenceTarget.ifSpatial { position = Vector3f(0.0f,0f,-1f) } - Thread.sleep(2) + val direction = (pointWorld - headCenter).normalize() + if (tracking) { + addSpine(headCenter, direction, volume,0.8f, volume.viewerState.currentTimepoint) } - } - - } - companion object { - //run function from here, it will automatically choose the volume for rendering, please give the correct location of volume - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - val sv = SciView.create() - val command = sv.scijavaContext!!.getService(CommandService::class.java) - - command.run(OpenDirofTif::class.java, true, - hashMapOf( - "file" to File("E:\\dataset\\Pdu_H2BeGFP_CAAXmCherry_0123_20130312_192018.corrected-histone"), - "onlyFirst" to 10 - )) - .get() - - val argmap = HashMap() - command.run(VRHeadSetTrackingDemo::class.java, true, argmap) - .get() + Thread.sleep(2) + } } } } \ No newline at end of file From f0721389864b5d9e0c6a8ef42d8b030c3772aa6a Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:01:50 +0200 Subject: [PATCH 52/55] Use lazylogger instead of Logservice --- .../iview/commands/demo/advanced/CellTrackingBase.kt | 10 +++++++++- .../sc/iview/commands/demo/advanced/TrackingTest.kt | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt index 93871111..694bf2a6 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt @@ -8,6 +8,7 @@ import graphics.scenery.primitives.Cylinder import graphics.scenery.utils.MaybeIntersects import graphics.scenery.utils.SystemHelpers import graphics.scenery.utils.extensions.minus +import graphics.scenery.utils.lazyLogger import graphics.scenery.volumes.RAIVolume import graphics.scenery.volumes.Volume import org.joml.Math @@ -27,7 +28,7 @@ import kotlin.concurrent.thread open class CellTrackingBase( open var sciview: SciView ) { - lateinit var logger: LogService + val logger by lazyLogger() lateinit var sessionId: String lateinit var sessionDirectory: Path @@ -49,6 +50,7 @@ open class CellTrackingBase( var volumeScaleFactor = 1.0f + // determines whether the volume and hedgehogs should keep listening for updates or not var cellTrackingActive: Boolean = false open var linkCreationCallback: ((HedgehogAnalysis.SpineGraphVertex) -> Unit)? = null @@ -62,6 +64,12 @@ open class CellTrackingBase( Backward } + init { + logger.info("we constructed celltrackingbase now. Lets toggle VR") + sciview.toggleVRRendering() + logger.info("yeah we toggled vr alright") + } + fun addHedgehog() { logger.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt index 0fc0a445..4d93f8e7 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TrackingTest.kt @@ -283,8 +283,8 @@ class TrackingTest( ) val count = samples.filterNotNull().count { it > 0.002f } - logger.info("count of samples: "+ count.toString()) - logger.info(samples) + logger.info("count of samples: $count") + logger.info(samples.joinToString { ", " }) // spine.metadata["spine"] = metadata // spine.instancedProperties["ModelMatrix"] = { spine.spatial().world } From 7c2f2c37fa0781ca9e0ea4e5cc59da0c803f4953 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:30:32 +0200 Subject: [PATCH 53/55] StartEyeTrackingDirectly: Rename to EyeTrackingCommand --- src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt index 88c812e5..6cfa3f45 100644 --- a/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt +++ b/src/test/kotlin/sc/iview/StartEyeTrackingDirectly.kt @@ -1,12 +1,10 @@ import graphics.scenery.utils.extensions.times -import graphics.scenery.utils.lazyLogger import graphics.scenery.volumes.RAIVolume import graphics.scenery.volumes.TransferFunction -import org.joml.Vector3f import org.scijava.command.CommandService import org.scijava.ui.UIService import sc.iview.SciView -import sc.iview.commands.demo.advanced.EyeTrackingDemo +import sc.iview.commands.demo.advanced.EyeTrackingCommand //object StartEye { @@ -35,7 +33,7 @@ fun main() { val command = sv.scijavaContext!!.getService(CommandService::class.java) val argmap = HashMap() - command.run(EyeTrackingDemo::class.java, true, argmap) + command.run(EyeTrackingCommand::class.java, true, argmap) } //} \ No newline at end of file From d2d261898cb64fb09bf1e89276f3bb12138a0aed Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:53:13 +0100 Subject: [PATCH 54/55] SciviewBridge: wrap eyeTracking into a separate thread to fix blank screen on toggleVRRendering --- .../sc/iview/commands/demo/advanced/CellTrackingBase.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt index 694bf2a6..1d69738d 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt @@ -64,12 +64,6 @@ open class CellTrackingBase( Backward } - init { - logger.info("we constructed celltrackingbase now. Lets toggle VR") - sciview.toggleVRRendering() - logger.info("yeah we toggled vr alright") - } - fun addHedgehog() { logger.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) From 317734bdfdcdb285d25e1e299ef56c67a7e31fa2 Mon Sep 17 00:00:00 2001 From: Samuel Pantze <83579186+smlpt@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:37:22 +0100 Subject: [PATCH 55/55] Add TimepointObserver to unify timepoint updates across VR and bridge --- .../demo/advanced/CellTrackingBase.kt | 26 +++++++++++++++---- .../commands/demo/advanced/EyeTracking.kt | 9 +++---- .../demo/advanced/TimepointObserver.kt | 13 ++++++++++ 3 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 src/main/kotlin/sc/iview/commands/demo/advanced/TimepointObserver.kt diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt index 1d69738d..40cb962c 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/CellTrackingBase.kt @@ -64,6 +64,22 @@ open class CellTrackingBase( Backward } + private val observers = mutableListOf() + + /** Registers a new observer that will get updated whenever the VR user triggers a timepoint update. */ + fun registerObserver(observer: TimepointObserver) { + observers.add(observer) + } + + /** Unregisters the timepoint observer. */ + fun unregisterObserver(observer: TimepointObserver) { + observers.remove(observer) + } + + private fun notifyObservers(timepoint: Int) { + observers.forEach { it.onTimePointChanged(timepoint) } + } + fun addHedgehog() { logger.info("added hedgehog") val hedgehog = Cylinder(0.005f, 1.0f, 16) @@ -234,19 +250,19 @@ open class CellTrackingBase( while(sciview.running && cellTrackingActive) { if(playing || skipToNext || skipToPrevious) { val oldTimepoint = volume.viewerState.currentTimepoint - val newVolume = if(skipToNext || playing) { + if (skipToNext || playing) { skipToNext = false if(direction == PlaybackDirection.Forward) { - volume.nextTimepoint() + notifyObservers(oldTimepoint + 1) } else { - volume.previousTimepoint() + notifyObservers(oldTimepoint - 1) } } else { skipToPrevious = false if(direction == PlaybackDirection.Forward) { - volume.previousTimepoint() + notifyObservers(oldTimepoint - 1) } else { - volume.nextTimepoint() + notifyObservers(oldTimepoint + 1) } } val newTimepoint = volume.viewerState.currentTimepoint diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt index 7e4d0716..f9adcf7f 100644 --- a/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/EyeTracking.kt @@ -279,18 +279,17 @@ class EyeTracking( } override fun stop() { - logger.info("Shutting down eye tracking environment. Disabling VR now...") + hmd.close() + logger.info("Shut down HMD and keybindings.") sciview.toggleVRRendering() - logger.info("Stopping volume and hedgehog updater thread...") + logger.info("Shut down eye tracking environment and disabled VR.") cellTrackingActive = false - logger.info("Deleting eye tracking scene objects...") + logger.info("Stopped volume and hedgehog updater thread.") lightTetrahedron.forEach { sciview.deleteNode(it) } sciview.deleteNode(sciview.find("Shell")) listOf(referenceTarget, calibrationTarget, laser, debugBoard, hedgehogs).forEach { sciview.deleteNode(it) } - logger.info("Shutting down HMD and keybindings...") - hmd.close() logger.info("Successfully cleaned up eye tracking environemt.") } diff --git a/src/main/kotlin/sc/iview/commands/demo/advanced/TimepointObserver.kt b/src/main/kotlin/sc/iview/commands/demo/advanced/TimepointObserver.kt new file mode 100644 index 00000000..130bc4ce --- /dev/null +++ b/src/main/kotlin/sc/iview/commands/demo/advanced/TimepointObserver.kt @@ -0,0 +1,13 @@ +package sc.iview.commands.demo.advanced + +/** + * Interface to allow subscription to timepoint updates, especially for updating sciview contents + * after a user triggered a timepoint change via controller input. + */ +interface TimepointObserver { + + /** + * Called when the timepoint was updated. + */ + fun onTimePointChanged(timepoint: Int) +} \ No newline at end of file