diff --git a/.github/workflows/dart.yaml b/.github/workflows/dart.yaml new file mode 100644 index 0000000..bf9c815 --- /dev/null +++ b/.github/workflows/dart.yaml @@ -0,0 +1,35 @@ +name: Dart Tests + +on: + pull_request: + branches: [master, main] + push: + branches: [master, main] + +jobs: + build: + name: Run Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dart-lang/setup-dart@v1 + with: + sdk: 3.0 + + - name: Install Requirements + working-directory: src/dart + run: dart pub get + + # Verify the use of 'dart format' on each commit. + - name: Check formatting + working-directory: src/dart + run: dart format --output=none --set-exit-if-changed . + + # Passing '--fatal-infos' for slightly stricter analysis. + - name: Analyze + working-directory: src/dart + run: dart analyze --fatal-infos + + - name: Run tests + working-directory: src/dart + run: dart test diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 8cf7d2c..8658793 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -1,11 +1,10 @@ -name: Build +name: Python Tests on: push: - branches: [ master, main ] + branches: [master, main] pull_request: - branches: [ master, main ] - + branches: [master, main] jobs: test: @@ -13,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.10" ] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 @@ -27,4 +26,8 @@ jobs: - name: Run tests working-directory: src/python - run: pytest pose_format \ No newline at end of file + run: pytest pose_format + + - name: Run additional tests + working-directory: src/python + run: pytest tests -s diff --git a/src/dart/.gitignore b/src/dart/.gitignore new file mode 100644 index 0000000..c029900 --- /dev/null +++ b/src/dart/.gitignore @@ -0,0 +1,6 @@ +.dart_tool/ +test.gif +pubspec.lock +analysis_options.yaml +pose_file.pose +demo.gif \ No newline at end of file diff --git a/src/dart/CHANGELOG.md b/src/dart/CHANGELOG.md new file mode 100644 index 0000000..c20c93d --- /dev/null +++ b/src/dart/CHANGELOG.md @@ -0,0 +1,37 @@ +## 1.0.0 + +- Can Load Pose File + +## 1.0.1 + +- Add Documentation + +## 1.0.2 + +- Add More Tests + +## 1.1.0 + +- Add Visualization + +## 1.1.1 + +- Fix Visualization + +## 1.1.2 + +- Removed Unused files +- Re-structre + +## 1.1.3 + +- Optimizations + +## 1.1.4 + +- Method Divide + +## 1.1.5 + +- Async Tasks + diff --git a/src/dart/LICENSE b/src/dart/LICENSE new file mode 100644 index 0000000..1e54c17 --- /dev/null +++ b/src/dart/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, the respective contributors, as shown by the AUTHORS file. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/src/dart/README.md b/src/dart/README.md new file mode 100644 index 0000000..6d3d34f --- /dev/null +++ b/src/dart/README.md @@ -0,0 +1,46 @@ +# Pose + +[![pub package](https://img.shields.io/pub/v/pose.svg)](https://pub.dev/packages/pose) + +This is `dart` implementation of its [python counterpart](https://github.com/sign-language-processing/pose/tree/master/src/python) with limited features + +This repository helps developers interested in Sign Language Processing (SLP) by providing a complete toolkit for working with poses. + +## File Format Structure + +The file format is designed to accommodate any pose type, an arbitrary number of people, and an indefinite number of frames. +Therefore it is also very suitable for video data, and not only single frames. + +At the core of the file format is `Header` and a `Body`. + +* The header for example contains the following information: + + - The total number of pose points. (How many points exist.) + - The exact positions of these points. (Where do they exist.) + - The connections between these points. (How are they connected.) + +## Features + +- ✔️ Reading +- ❌ Normalizing +- ❌ Augmentation +- ❌ Interpolation +- ✔️ Visualization (2x slow compared to python and supports only GIF) + +## Usage + +```dart +import 'dart:io'; +import 'dart:typed_data'; +import 'package:pose/pose.dart'; + +void main() async { + File file = File("pose_file.pose"); + Uint8List fileContent = file.readAsBytesSync(); + Pose pose = Pose.read(fileContent); + PoseVisualizer p = PoseVisualizer(pose); + await p.saveGif("demo.gif", p.draw()); +} +``` + +![Demo Gif](https://raw.githubusercontent.com/sign-language-processing/pose/master/src/dart/assets/demo.gif) diff --git a/src/dart/example/pose_example.dart b/src/dart/example/pose_example.dart new file mode 100644 index 0000000..c6e0fc8 --- /dev/null +++ b/src/dart/example/pose_example.dart @@ -0,0 +1,22 @@ +import 'dart:io'; +import 'dart:typed_data'; +import 'package:pose/pose.dart'; + +void main() async { + Stopwatch stopwatch = Stopwatch()..start(); + + File file = File("pose_file.pose"); + Uint8List fileContent = file.readAsBytesSync(); + print("File Read"); + + Pose pose = Pose.read(fileContent); + print("File Loaded"); + + PoseVisualizer p = PoseVisualizer(pose, thickness: 2); + print("File Visualized"); + + File giffile = await p.saveGif("demo.gif", p.draw()); + print("File Saved ${giffile.path}"); + + print('Time taken : ${stopwatch.elapsed}'); +} diff --git a/src/dart/lib/numdart.dart b/src/dart/lib/numdart.dart new file mode 100644 index 0000000..6fa3a1f --- /dev/null +++ b/src/dart/lib/numdart.dart @@ -0,0 +1,210 @@ +// ignore_for_file: non_constant_identifier_names + +import 'dart:math' as math; +import 'dart:typed_data'; + +/// Represents a structure with a specified format and size. +class Struct { + final String format; + final int size; + + Struct(this.format, this.size); +} + +/// Contains predefined Struct objects for commonly used data types. +class ConstStructs { + static final Struct float = Struct(" bytesData) { + int intValue = 0; + for (int i = 0; i < bytesData.length; i++) { + intValue += bytesData[i] << (i * 8); + } + final int sign = (intValue & (1 << (8 * bytesData.length - 1))) != 0 ? -1 : 1; + final int exponent = ((intValue >> 23) & 0xFF) - 127; + final int mantissa = (intValue & 0x7FFFFF) | 0x800000; + final num result = sign * mantissa * math.pow(2, exponent - 23); + return result.toDouble(); +} + +/// Converts bytes to an integer. +int bytesToInt(List bytesData, + {bool signed = false, Endian byteOrder = Endian.little}) { + final ByteData byteData = ByteData.sublistView(Uint8List.fromList(bytesData)); + if (signed) { + switch (bytesData.length) { + case 1: + return byteData.getInt8(0); + case 2: + return byteData.getInt16(0, byteOrder); + case 4: + return byteData.getInt32(0, byteOrder); + case 8: + return byteData.getInt64(0, byteOrder); + default: + throw ArgumentError('Invalid byte length for signed integer'); + } + } else { + switch (bytesData.length) { + case 1: + return byteData.getUint8(0); + case 2: + return byteData.getUint16(0, byteOrder); + case 4: + return byteData.getUint32(0, byteOrder); + case 8: + return byteData.getUint64(0, byteOrder); + default: + throw ArgumentError('Invalid byte length for unsigned integer'); + } + } +} + +/// Calculates the product of a sequence of integers. +int prod(List seq) { + int result = 1; + for (int num in seq) { + result *= num; + } + return result; +} + +/// Represents a function that converts bytes to a numeric value. +typedef NumConversionFunction = num Function(List); + +/// Constructs an n-dimensional array from a buffer based on the given shape and format. +List ndarray(List shape, Struct s, List buffer, int offset) { + NumConversionFunction func; + if (s.format == " matrix = []; + + if (shape.length == 2) { + for (int i = 0; i < shape[0]; i++) { + List row = []; + for (int j = 0; j < shape[1]; j++) { + row.add(func(buffer.sublist(offset, offset + s.size))); + offset += s.size; + } + matrix.add(row); + } + } else if (shape.length == 3) { + for (int i = 0; i < shape[0]; i++) { + List innerMatrix = []; + for (int j = 0; j < shape[1]; j++) { + List row = []; + for (int k = 0; k < shape[2]; k++) { + row.add(func(buffer.sublist(offset, offset + s.size))); + offset += s.size; + } + innerMatrix.add(row); + } + matrix.add(innerMatrix); + } + } else if (shape.length == 4) { + for (int i = 0; i < shape[0]; i++) { + List innerMatrix1 = []; + for (int j = 0; j < shape[1]; j++) { + List innerMatrix2 = []; + for (int k = 0; k < shape[2]; k++) { + List innerMatrix3 = []; + for (int l = 0; l < shape[3]; l++) { + innerMatrix3.add(func(buffer.sublist(offset, offset + s.size))); + offset += s.size; + } + innerMatrix2.add(innerMatrix3); + } + innerMatrix1.add(innerMatrix2); + } + matrix.add(innerMatrix1); + } + } else { + throw ArgumentError("Shape length must be 2, 3, or 4."); + } + + return matrix; +} + +/// Computes the mean along a specified axis. +List mean(List> values, {int? axis}) { + if (values.isEmpty) { + return [double.nan]; // Return NaN for empty lists + } + + if (axis == null) { + final List flattenedValues = values.expand((list) => list).toList(); + final num total = flattenedValues.reduce((a, b) => a + b); + return [total / flattenedValues.length]; + } else if (axis == 0) { + final List columnSums = List.filled(values[0].length, 0); + for (List row in values) { + for (int i = 0; i < row.length; i++) { + columnSums[i] += row[i]; + } + } + return columnSums.map((sum) => sum / values.length).toList(); + } else if (axis == 1) { + return values + .map((row) => row.reduce((a, b) => a + b) / row.length) + .toList(); + } else { + throw ArgumentError("Axis must be null, 0, or 1."); + } +} + +/// Represents a masked array with data and mask. +class MaskedArray { + final List data; + final List> mask; + + MaskedArray(this.data, this.mask); + + /// Rounds the data values in the masked array. + MaskedArray rint() { + final List> roundedData = []; + for (int i = 0; i < data.length; i++) { + List row = []; + for (int j = 0; j < data[i].length; j++) { + if (mask[i][j] != 0) { + row.add(_round(data[i][j])); + } else { + row.add(data[i][j]); + } + } + roundedData.add(row); + } + return MaskedArray(roundedData, mask); + } + + List round() { + return _roundList(data); + } + + dynamic _round(dynamic elem) { + if (elem is List) { + return _roundList(elem); + } else { + return (elem).round(); + } + } + + List _roundList(List elem) { + List roundedList = []; + for (int i = 0; i < elem.length; i++) { + roundedList.add(_round(elem[i])); + } + return roundedList; + } +} diff --git a/src/dart/lib/pose.dart b/src/dart/lib/pose.dart new file mode 100644 index 0000000..75b1b88 --- /dev/null +++ b/src/dart/lib/pose.dart @@ -0,0 +1,4 @@ +library; + +export 'src/pose.dart'; +export 'src/pose_visualizer.dart'; diff --git a/src/dart/lib/reader.dart b/src/dart/lib/reader.dart new file mode 100644 index 0000000..d5ca920 --- /dev/null +++ b/src/dart/lib/reader.dart @@ -0,0 +1,70 @@ +import 'dart:convert'; +import 'dart:typed_data'; +import 'package:pose/numdart.dart' show Struct, ConstStructs; +import 'package:pose/numdart.dart' as nd; + +/// Class for reading data from a byte buffer. +class BufferReader { + final Uint8List buffer; + int readOffset; + + /// Constructs a BufferReader with the given byte buffer. + BufferReader(this.buffer) : readOffset = 0; + + /// Returns the number of bytes left to read from the buffer. + int bytesLeft() { + return buffer.length - readOffset; + } + + /// Reads a fixed-size chunk of bytes from the buffer. + Uint8List unpackF(int size) { + final Uint8List data = buffer.sublist(readOffset, readOffset + size); + advance(Struct("", size)); + return data; + } + + /// Reads numeric data from the buffer and constructs an n-dimensional array. + List unpackNum(Struct s, List shape) { + final List arr = nd.ndarray(shape, s, buffer, readOffset); + final int arrayBufferSize = nd.prod(shape); + advance(s, arrayBufferSize); + return arr; + } + + /// Unpacks a single value from the buffer based on the given format. + dynamic unpack(Struct s) { + final Uint8List data = buffer.sublist(readOffset, readOffset + s.size); + advance(s); + + final List result = []; + + if (s.format == " confidence; + + /// Constructor for PoseBody. + /// + /// Takes [fps], [data], and [confidence] as parameters. + PoseBody(this.fps, this.data, this.confidence); + + /// Reads pose body data based on the provided header and reader. + /// + /// Returns a PoseBody instance. + static PoseBody read( + PoseHeader header, BufferReader reader, Map kwargs) { + if ((header.version * 1000).round() == 100) { + return read_v0_1(header, reader, kwargs); + } + + throw UnimplementedError("Unknown version - ${header.version}"); + } + + /// Reads pose body data for version 0.1. + /// + /// Takes [header], [reader], and [kwargs] as parameters. + /// Returns the read data. + static dynamic read_v0_1( + PoseHeader header, BufferReader reader, Map kwargs) { + final List lst = reader.unpack(ConstStructs.double_ushort); + final int _people = reader.unpack(ConstStructs.ushort); + final int fps = lst[0]; + int _frames = lst[1]; + + final int _points = + header.components.map((c) => c.points.length).reduce((a, b) => a + b); + final int _dims = header.components + .map((c) => c.format.length) + .reduce((a, b) => a > b ? a : b) - + 1; + _frames = reader.bytesLeft() ~/ (_people * _points * (_dims + 1) * 4); + + final List data = read_v0_1_frames(_frames, [_people, _points, _dims], + reader, kwargs['startFrame'], kwargs['endFrame']); + final List confidence = read_v0_1_frames(_frames, [_people, _points], + reader, kwargs['startFrame'], kwargs['endFrame']); + + return PoseBody(fps.toDouble(), data, confidence); + } + + /// Reads pose body data for version 0.1 frames. + /// + /// Takes [frames], [shape], [reader], [startFrame], and [endFrame] as parameters. + /// Returns the read data. + static dynamic read_v0_1_frames(int frames, List shape, + BufferReader reader, int? startFrame, int? endFrame) { + final Struct s = ConstStructs.float; + int _frames = frames; + + if (startFrame != null && startFrame > 0) { + if (startFrame >= frames) { + throw ArgumentError("Start frame is greater than the number of frames"); + } + reader.advance(s, (startFrame * shape.reduce((a, b) => a * b))); + _frames -= startFrame; + } + + int removeFrames = 0; + if (endFrame != null) { + endFrame = endFrame > frames ? frames : endFrame; + removeFrames = frames - endFrame; + _frames -= removeFrames; + } + + final List tensor = + reader.unpackNum(ConstStructs.float, [_frames, ...shape]); + if (removeFrames != 0) { + reader.advance(s, (removeFrames * shape.reduce((a, b) => a * b))); + } + + return tensor; + } +} diff --git a/src/dart/lib/src/pose_header.dart b/src/dart/lib/src/pose_header.dart new file mode 100644 index 0000000..d410367 --- /dev/null +++ b/src/dart/lib/src/pose_header.dart @@ -0,0 +1,119 @@ +import 'dart:math'; +import 'package:pose/reader.dart'; +import 'package:pose/numdart.dart'; + +/// Class representing a component of a pose header. +/// +/// This class contains information about the points, limbs, colors, and format of a pose header component. +class PoseHeaderComponent { + final String name; + final List points; + final List> limbs; + final List> colors; + final String format; + late List relativeLimbs; + + /// Constructor for PoseHeaderComponent. + /// + /// Takes [name], [points], [limbs], [colors], and [format] as parameters. + PoseHeaderComponent( + this.name, this.points, this.limbs, this.colors, this.format) { + relativeLimbs = getRelativeLimbs(); + } + + /// Reads a PoseHeaderComponent from the reader based on the specified version. + /// + /// Takes [version] and [reader] as parameters. + /// Returns a PoseHeaderComponent instance. + static PoseHeaderComponent read(double version, BufferReader reader) { + final String name = reader.unpackStr(); + final String pointFormat = reader.unpackStr(); + final int pointsCount = reader.unpack(ConstStructs.ushort); + final int limbsCount = reader.unpack(ConstStructs.ushort); + final int colorsCount = reader.unpack(ConstStructs.ushort); + final List points = + List.generate(pointsCount, (_) => reader.unpackStr()); + final List> limbs = List.generate( + limbsCount, + (_) => Point(reader.unpack(ConstStructs.ushort), + reader.unpack(ConstStructs.ushort))); + final List> colors = List.generate( + colorsCount, + (_) => [ + reader.unpack(ConstStructs.ushort), + reader.unpack(ConstStructs.ushort), + reader.unpack(ConstStructs.ushort) + ], + ); + + return PoseHeaderComponent(name, points, limbs, colors, pointFormat); + } + + /// Calculates the relative limbs for the component. + /// + /// Returns a list of relative limbs. + List getRelativeLimbs() { + final Map limbsMap = {}; + for (int i = 0; i < limbs.length; i++) { + limbsMap[limbs[i].y] = i; + } + return limbs.map((limb) => limbsMap[limb.x]).toList(); + } +} + +/// Class representing dimensions of a pose header. +/// +/// This class contains information about the width, height, and depth of a pose header. +class PoseHeaderDimensions { + final int width; + final int height; + final int depth; + + /// Constructor for PoseHeaderDimensions. + /// + /// Takes [width], [height], and [depth] as parameters. + PoseHeaderDimensions(this.width, this.height, this.depth); + + /// Reads PoseHeaderDimensions from the reader based on the specified version. + /// + /// Takes [version] and [reader] as parameters. + /// Returns a PoseHeaderDimensions instance. + static PoseHeaderDimensions read(double version, BufferReader reader) { + final int width = reader.unpack(ConstStructs.ushort); + final int height = reader.unpack(ConstStructs.ushort); + final int depth = reader.unpack(ConstStructs.ushort); + + return PoseHeaderDimensions(width, height, depth); + } +} + +/// Class representing a pose header. +/// +/// This class contains information about the version, dimensions, components, and bounding box status of a pose header. +class PoseHeader { + final double version; + final PoseHeaderDimensions dimensions; + final List components; + final bool isBbox; + + /// Constructor for PoseHeader. + /// + /// Takes [version], [dimensions], [components], and [isBbox] as parameters. + PoseHeader(this.version, this.dimensions, this.components, + {this.isBbox = false}); + + /// Reads a PoseHeader from the reader. + /// + /// Takes [reader] as a parameter. + /// Returns a PoseHeader instance. + static PoseHeader read(BufferReader reader) { + final double version = reader.unpack(ConstStructs.float); + final PoseHeaderDimensions dimensions = + PoseHeaderDimensions.read(version, reader); + final int componentsCount = reader.unpack(ConstStructs.ushort); + final List components = List.generate( + componentsCount, (_) => PoseHeaderComponent.read(version, reader)); + + return PoseHeader(version, dimensions, components); + } +} diff --git a/src/dart/lib/src/pose_visualizer.dart b/src/dart/lib/src/pose_visualizer.dart new file mode 100644 index 0000000..e58318f --- /dev/null +++ b/src/dart/lib/src/pose_visualizer.dart @@ -0,0 +1,239 @@ +// ignore_for_file: no_leading_underscores_for_local_identifiers + +import 'dart:io'; +import 'dart:math'; +import 'dart:typed_data'; +import 'package:image/image.dart'; +import 'package:pose/pose.dart'; +import 'package:pose/numdart.dart' as nd; +import 'package:pose/numdart.dart' show MaskedArray; +import 'package:pose/src/pose_header.dart'; +import 'package:tuple/tuple.dart'; + +/// Class responsible for visualizing poses. +class PoseVisualizer { + final Pose pose; + int? thickness; + late double fps; + Image? background; + + /// Constructs a PoseVisualizer with the given pose and optional thickness. + PoseVisualizer(this.pose, {this.thickness}) : fps = pose.body.fps; + + /// Draws a single frame of the pose on the given image. + Image _drawFrame(MaskedArray frame, List frameConfidence, Image img) { + final Pixel pixelColor = img.getPixel(0, 0); + final Tuple3 backgroundColor = + Tuple3.fromList( + [pixelColor.r, pixelColor.g, pixelColor.b]); + + thickness ??= (sqrt(img.width * img.height) / 150).round(); + final int radius = (thickness! / 2).round(); + + for (int i = 0; i < frame.data.length; i++) { + final List person = frame.data[i]; + final List personConfidence = frameConfidence[i]; + + final List> points2D = List>.from( + person.map((p) => Tuple2(p[0], p[1]))); + + int idx = 0; + for (PoseHeaderComponent component in pose.header.components) { + final List> colors = [ + for (List c in component.colors) + Tuple3.fromList(c) // can be reversed + ]; + + Tuple3 _pointColor(int pI) { + final double opacity = personConfidence[pI + idx]; + final List nColor = colors[pI % component.colors.length] + .toList() + .map((e) => (e * opacity).toInt()) + .toList(); + final List newColor = backgroundColor + .toList() + .map((e) => (e * (1 - opacity)).toInt()) + .toList(); + + final Tuple3 ndColor = Tuple3.fromList([ + for (int i in Iterable.generate(nColor.length)) + (nColor[i] + newColor[i]) + ]); + return ndColor; + } + + // Draw Points + for (int i = 0; i < component.points.length; i++) { + if (personConfidence[i + idx] > 0) { + final Tuple2 center = + Tuple2.fromList(person[i + idx].take(2).toList()); + final Tuple3 colorTuple = _pointColor(i); + + drawCircle( + img, + x: center.item1, + y: center.item2, + radius: radius, + color: ColorFloat16.fromList([ + colorTuple.item1, + colorTuple.item2, + colorTuple.item3 + ].map((e) => (e.toDouble())).toList()), + ); + } + } + + if (pose.header.isBbox) { + final Tuple2 point1 = points2D[0 + idx]; + final Tuple2 point2 = points2D[1 + idx]; + + final Tuple3 temp1 = _pointColor(0); + final Tuple3 temp2 = _pointColor(1); + + drawRect(img, + x1: point1.item1, + y1: point1.item2, + x2: point2.item1, + y2: point2.item2, + color: ColorFloat16.fromList(nd.mean([ + [temp1.item1, temp1.item2, temp1.item3], + [temp2.item1, temp2.item2, temp2.item3] + ], axis: 0)), + thickness: thickness!); + } else { + // Draw Limbs + for (var limb in component.limbs) { + if (personConfidence[limb.x + idx] > 0 && + personConfidence[limb.y + idx] > 0) { + final Tuple2 point1 = points2D[limb.x + idx]; + final Tuple2 point2 = points2D[limb.y + idx]; + + final Tuple3 temp1 = _pointColor(limb.x); + final Tuple3 temp2 = _pointColor(limb.y); + + drawLine(img, + x1: point1.item1, + y1: point1.item2, + x2: point2.item1, + y2: point2.item2, + color: ColorFloat16.fromList(nd.mean([ + [temp1.item1, temp1.item2, temp1.item3], + [temp2.item1, temp2.item2, temp2.item3] + ], axis: 0)), + thickness: thickness!); + } + } + } + + idx += component.points.length; + } + } + + return img; + } + + /// Generates frames for the pose visualization. + Stream draw( + {List backgroundColor = const [0, 0, 0], int? maxFrames}) async* { + final List intFrames = MaskedArray(pose.body.data, []).round(); + + final background = Image( + width: pose.header.dimensions.width, + height: pose.header.dimensions.height, + backgroundColor: ColorFloat16.fromList(backgroundColor), + ); + + for (int i = 0; + i < min(intFrames.length, maxFrames ?? intFrames.length); + i++) { + yield _drawFrame(MaskedArray(intFrames[i], []), pose.body.confidence[i], + background.clone()); + } + } + + // Generate GIF from frames + Future generateGif(Stream frames, {double fps = 24}) async { + final int frameDuration = (100 / fps).round(); + final GifEncoder encoder = GifEncoder(delay: 0, repeat: 0); + + await for (Image frame in frames) { + encoder.addFrame(frame, duration: frameDuration); + } + + final Uint8List? image = encoder.finish(); + if (image != null) { + return image; + } + + throw Exception('Failed to encode GIF.'); + } + + /// Saves the visualization as a GIF. + Future saveGif(String fileName, Stream frames, + {double fps = 24}) async { + Uint8List image = await generateGif(frames, fps: fps); + return await File(fileName).writeAsBytes(image); + } +} + +class FastAndUglyPoseVisualizer extends PoseVisualizer { + FastAndUglyPoseVisualizer(Pose pose, {int? thickness}) + : super(pose, thickness: thickness); + + Image _uglyDrawFrame(MaskedArray frame, Image img, int color) { + final Tuple2 ignoredPoint = Tuple2.fromList([0, 0]); + + // Note: this can be made faster by drawing polylines instead of lines + final thickness = 1; + + for (int i = 0; i < frame.data.length; i++) { + final List person = frame.data[i]; + + final List> points2D = List>.from( + person.map((p) => Tuple2(p[0], p[1]))); + + int idx = 0; + for (PoseHeaderComponent component in pose.header.components) { + for (var limb in component.limbs) { + final Tuple2 point1 = points2D[limb.x + idx]; + final Tuple2 point2 = points2D[limb.y + idx]; + + if (point1 != ignoredPoint && point2 != ignoredPoint) { + // Antialiasing is a bit slow, but necessary + drawLine( + img, + x1: point1.item1, + y1: point1.item2, + x2: point2.item1, + y2: point2.item2, + antialias: true, + color: ColorFloat16.fromList([color.toDouble()]), + thickness: thickness, + ); + } + } + idx += component.points.length; + } + } + return img; + } + + Stream uglyDraw( + {int backgroundColor = 0, int foregroundColor = 255}) async* { + final List intFrames = MaskedArray(pose.body.data, []).round(); + + final background = Image( + width: pose.header.dimensions.width, + height: pose.header.dimensions.height, + backgroundColor: ColorFloat16.fromList([backgroundColor.toDouble()]), + ); + + for (int i = 0; i < intFrames.length; i++) { + yield _uglyDrawFrame( + MaskedArray(intFrames[i], []), + background.clone(), + foregroundColor, + ); + } + } +} diff --git a/src/dart/pubspec.yaml b/src/dart/pubspec.yaml new file mode 100644 index 0000000..09b11e1 --- /dev/null +++ b/src/dart/pubspec.yaml @@ -0,0 +1,17 @@ +name: pose +description: Dart library for loading, viewing, augmenting, and handling .pose files +version: 1.1.5 +repository: https://github.com/bipinkrish/pose +topics: + - pose + +environment: + sdk: ^3.0.0 + +dependencies: + image: ^4.1.7 + tuple: ^2.0.2 + +dev_dependencies: + lints: ^2.0.0 + test: ^1.21.0 \ No newline at end of file diff --git a/src/dart/test/data/mediapipe.pose b/src/dart/test/data/mediapipe.pose new file mode 100644 index 0000000..f32f98d Binary files /dev/null and b/src/dart/test/data/mediapipe.pose differ diff --git a/src/dart/test/data/mediapipe_hand_normalized.pose b/src/dart/test/data/mediapipe_hand_normalized.pose new file mode 100644 index 0000000..e6eed7d Binary files /dev/null and b/src/dart/test/data/mediapipe_hand_normalized.pose differ diff --git a/src/dart/test/data/mediapipe_long.pose b/src/dart/test/data/mediapipe_long.pose new file mode 100644 index 0000000..5c008bc Binary files /dev/null and b/src/dart/test/data/mediapipe_long.pose differ diff --git a/src/dart/test/data/mediapipe_long_hand_normalized.pose b/src/dart/test/data/mediapipe_long_hand_normalized.pose new file mode 100644 index 0000000..9731534 Binary files /dev/null and b/src/dart/test/data/mediapipe_long_hand_normalized.pose differ diff --git a/src/dart/test/pose_test.dart b/src/dart/test/pose_test.dart new file mode 100644 index 0000000..4dfb0db --- /dev/null +++ b/src/dart/test/pose_test.dart @@ -0,0 +1,104 @@ +import 'dart:io'; +import 'dart:typed_data'; +import 'package:pose/pose.dart'; +import 'package:pose/src/pose_header.dart'; +import 'package:test/test.dart'; + +void main() { + Pose getPose(String filePath) { + File file = File(filePath); + Uint8List fileContent = file.readAsBytesSync(); + return Pose.read(fileContent); + } + + group('Pose Tests', () { + test("Mediapipe", () { + Pose pose = getPose("test/data/mediapipe.pose"); + + List confidence = pose.body.confidence; + expect((confidence.length, confidence[0].length, confidence[0][0].length), + equals((170, 1, 178))); + + List data = pose.body.data; + expect(( + data.length, + data[0].length, + data[0][0].length, + data[0][0][0].length + ), equals((170, 1, 178, 3))); + + PoseHeaderDimensions dimensions = pose.header.dimensions; + expect((dimensions.depth, dimensions.width, dimensions.height), + equals((640, 1250, 1250))); + + expect(pose.body.fps, equals(24.0)); + expect(pose.header.version, equals(0.10000000149011612)); + }); + test("Mediapipe long", () { + Pose pose = getPose("test/data/mediapipe_long.pose"); + + List confidence = pose.body.confidence; + expect((confidence.length, confidence[0].length, confidence[0][0].length), + equals((278, 1, 178))); + + List data = pose.body.data; + expect(( + data.length, + data[0].length, + data[0][0].length, + data[0][0][0].length + ), equals((278, 1, 178, 3))); + + PoseHeaderDimensions dimensions = pose.header.dimensions; + expect((dimensions.depth, dimensions.width, dimensions.height), + equals((640, 1250, 1250))); + + expect(pose.body.fps, equals(24.0)); + expect(pose.header.version, equals(0.10000000149011612)); + }); + test("Mediapipe hand normalized", () { + Pose pose = getPose("test/data/mediapipe_hand_normalized.pose"); + + List confidence = pose.body.confidence; + expect((confidence.length, confidence[0].length, confidence[0][0].length), + equals((170, 1, 21))); + + List data = pose.body.data; + expect(( + data.length, + data[0].length, + data[0][0].length, + data[0][0][0].length + ), equals((170, 1, 21, 3))); + + PoseHeaderDimensions dimensions = pose.header.dimensions; + expect((dimensions.depth, dimensions.width, dimensions.height), + equals((1, 223, 229))); + + expect(pose.body.fps, equals(24.0)); + expect(pose.header.version, equals(0.10000000149011612)); + }); + test("Mediapipe long hand normalized", () { + Pose pose = getPose("test/data/mediapipe_long_hand_normalized.pose"); + + List confidence = pose.body.confidence; + expect((confidence.length, confidence[0].length, confidence[0][0].length), + equals((278, 1, 21))); + + List data = pose.body.data; + expect(( + data.length, + data[0].length, + data[0][0].length, + data[0][0][0].length + ), equals((278, 1, 21, 3))); + + PoseHeaderDimensions dimensions = pose.header.dimensions; + expect((dimensions.depth, dimensions.width, dimensions.height), + equals((1, 221, 229))); + + expect(pose.body.fps, equals(24.0)); + expect(pose.header.version, equals(0.10000000149011612)); + }); + }); +} diff --git a/src/dart/test/reader_test.dart b/src/dart/test/reader_test.dart new file mode 100644 index 0000000..a4366a5 --- /dev/null +++ b/src/dart/test/reader_test.dart @@ -0,0 +1,91 @@ +import 'dart:convert'; +import 'dart:typed_data'; +import 'package:test/test.dart'; +import 'package:pose/reader.dart'; +import 'package:pose/numdart.dart'; + +void main() { + group('BufferReader Tests', () { + test('Advance Test', () { + BufferReader reader = BufferReader(Uint8List(0)); + reader.advance(ConstStructs.float, 10); + expect(reader.readOffset, equals(40)); + }); + + test('Unpack float Test', () { + double expected = 5.5; + Uint8List buffer = Uint8List.fromList( + [0x00, 0x00, 0xb0, 0x40]); // Equivalent to struct.pack(" stringBytes = utf8.encode(expected); + int length = stringBytes.length; + Uint8List buffer = Uint8List.fromList([ + length & 0xFF, // Low byte of the length + (length >> 8) & 0xFF, // High byte of the length + ...stringBytes // String data bytes + ]); // Equivalent to struct.pack("> expected = [ + [1.0, 2.5], + [3.5, 4.5] + ]; + + List> modifiedExpected = [ + [0.9, 2.5], + [3.5, 4.5] + ]; // Expected result after modification + + Uint8List buffer = Uint8List.fromList([ + 0x00, + 0x00, + 0x80, + 0x3f, + 0x00, + 0x00, + 0x20, + 0x40, + 0x00, + 0x00, + 0x60, + 0x40, + 0x00, + 0x00, + 0x90, + 0x40 + ]); // Equivalent to struct.pack(" unpacked = reader.unpackNum(ConstStructs.float, [2, 2]); + expect(unpacked, equals(expected)); + + unpacked[0][0] -= 0.1; // Modify the first element of the array + expect(unpacked, equals(modifiedExpected)); + }); + }); +} + +// print Uint8List in string format +String formatBytes(Uint8List bytes) { + StringBuffer buffer = StringBuffer('b\''); + for (int byte in bytes) { + if (byte >= 32 && byte <= 126) { + buffer.write(String.fromCharCode(byte)); + } else { + buffer.write('\\x${byte.toRadixString(16).padLeft(2, '0')}'); + } + } + buffer.write('\''); + return buffer.toString(); +} diff --git a/src/dart/test/visualization_test.dart b/src/dart/test/visualization_test.dart new file mode 100644 index 0000000..4cef2e6 --- /dev/null +++ b/src/dart/test/visualization_test.dart @@ -0,0 +1,26 @@ +// ignore_for_file: unused_local_variable + +import 'dart:io'; +import 'dart:typed_data'; +import 'package:pose/pose.dart'; +import 'package:test/test.dart'; + +void main() { + Pose getPose(String filePath) { + File file = File(filePath); + Uint8List fileContent = file.readAsBytesSync(); + return Pose.read(fileContent); + } + + group('Visualization Tests', () { + test("Mediapipe", () async { + try { + Pose pose = getPose("test/data/mediapipe.pose"); + PoseVisualizer p = PoseVisualizer(pose); + File file = await p.saveGif("test.gif", p.draw()); + } catch (e) { + Error(); + } + }, timeout: Timeout(Duration(minutes: 3))); + }); +} diff --git a/src/python/pose_format/pose_visualizer.py b/src/python/pose_format/pose_visualizer.py index 6c09a20..90317f3 100644 --- a/src/python/pose_format/pose_visualizer.py +++ b/src/python/pose_format/pose_visualizer.py @@ -39,7 +39,7 @@ def __init__(self, pose: Pose, thickness=None): except ImportError: raise ImportError("Please install OpenCV with: pip install opencv-python") - def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img) -> np.ndarray: + def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img, transparency: bool = False) -> np.ndarray: """ Draw frame of pose data of an image. @@ -51,6 +51,8 @@ def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img) Confidence values for each point in the frame. img : np.ndarray Background image where upon pose will be drawn. + transparency : bool + transparency decides opacity of background color, Returns ------- @@ -74,7 +76,9 @@ def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img) @lru_cache(maxsize=None) def _point_color(p_i: int): opacity = c[p_i + idx] - np_color = colors[p_i % len(component.colors)] * opacity + (1 - opacity) * background_color + np_color = colors[p_i % len(component.colors)] * opacity + (1 - opacity) * background_color[:3] # [:3] ignores alpha value if present + if transparency: + np_color = np.append(np_color, opacity * 255) return tuple([int(c) for c in np_color]) # Draw Points @@ -110,7 +114,7 @@ def _point_color(p_i: int): return img - def draw(self, background_color: Tuple[int, int, int] = (255, 255, 255), max_frames: int = None): + def draw(self, background_color: Tuple[int, int, int] = (255, 255, 255), max_frames: int = None, transparency: bool = False): """ draws pose on plain background using the specified color - for a number of frames. @@ -120,20 +124,22 @@ def draw(self, background_color: Tuple[int, int, int] = (255, 255, 255), max_fra RGB value for background color, default is white (255, 255, 255). max_frames : int, optional Maximum number of frames to process, if it is None, it processes all frames. - + transparency : bool + transparency decides opacity of background color, it is only used in the case of PNG i.e It doesn't affect GIF. Yields ------ np.ndarray Frames with the pose data drawn on a custom background color. - """ # ... + if transparency: + background_color += (0,) int_frames = np.array(np.around(self.pose.body.data.data), dtype="int32") - background = np.full((self.pose.header.dimensions.height, self.pose.header.dimensions.width, 3), + background = np.full((self.pose.header.dimensions.height, self.pose.header.dimensions.width, len(background_color)), fill_value=background_color, dtype="uint8") for frame, confidence in itertools.islice(zip(int_frames, self.pose.body.confidence), max_frames): - yield self._draw_frame(frame, confidence, img=background.copy()) + yield self._draw_frame(frame, confidence, img=background.copy(), transparency=transparency) def draw_on_video(self, background_video, max_frames: int = None, blur=False): """ @@ -203,16 +209,20 @@ def save_frame(self, f_name: str, frame: np.ndarray): """ self.cv2.imwrite(f_name, frame) - def save_gif(self, f_name: str, frames: Iterable[np.ndarray]): + def _save_image(self, f_name: str, frames: Iterable[np.ndarray], format: str = "GIF", transparency: bool = False): """ - Save pose frames as GIF. + Save pose frames as Image (GIF or PNG). Parameters ---------- f_name : str - filename to save GIF to. + filename to save Image to. frames : Iterable[np.ndarray] - Series of pose frames to be included in GIF. + Series of pose frames to be included in Image. + format : str + format to save takes either GIF or PNG. + transparency : bool + transparency decides opacity of background color. Returns ------- @@ -228,14 +238,66 @@ def save_gif(self, f_name: str, frames: Iterable[np.ndarray]): except ImportError: raise ImportError("Please install Pillow with: pip install Pillow") - images = [Image.fromarray(self.cv2.cvtColor(frame, self.cv2.COLOR_BGR2RGB)) for frame in frames] + if transparency: + cv_code = self.cv2.COLOR_BGR2RGBA + else: + cv_code = self.cv2.COLOR_BGR2RGB + + images = [Image.fromarray(self.cv2.cvtColor(frame, cv_code)) for frame in frames] images[0].save(f_name, - format="GIF", - append_images=images, + format=format, + append_images=images[1:], save_all=True, duration=1000 / self.pose.body.fps, - loop=0) + loop=0, + disposal=2 if transparency else 0) + + def save_gif(self, f_name: str, frames: Iterable[np.ndarray]): + """ + Save pose frames as GIF. + + Parameters + ---------- + f_name : str + filename to save GIF to. + frames : Iterable[np.ndarray] + Series of pose frames to be included in GIF. + + Returns + ------- + None + Raises + ------ + ImportError + If Pillow is not installed. + """ + self._save_image(f_name, frames, "GIF", False) + + def save_png(self, f_name: str, frames: Iterable[np.ndarray], transparency: bool = True): + """ + Save pose frames as PNG. + + Parameters + ---------- + f_name : str + filename to save PNG to. + frames : Iterable[np.ndarray] + Series of pose frames to be included in PNG. + transparency : bool + transparency decides opacity of background color. + + Returns + ------- + None + + Raises + ------ + ImportError + If Pillow is not installed. + """ + self._save_image(f_name, frames, "PNG", transparency) + def save_video(self, f_name: str, frames: Iterable[np.ndarray], custom_ffmpeg=None): """ Save pose frames as a video. diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml index ac0cc21..02db4b3 100644 --- a/src/python/pyproject.toml +++ b/src/python/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "scipy", "tqdm" ] +requires-python = ">= 3.9" [project.optional-dependencies] dev = [ diff --git a/src/python/tests/hand_normalization_test.py b/src/python/tests/hand_normalization_test.py index 8fd534d..11103bd 100644 --- a/src/python/tests/hand_normalization_test.py +++ b/src/python/tests/hand_normalization_test.py @@ -65,7 +65,7 @@ def test_hand_normalization(self): """ Test the normalization of hand pose data using the PoseNormalizer. """ - with open('data/mediapipe.pose', 'rb') as f: + with open('tests/data/mediapipe.pose', 'rb') as f: pose = Pose.read(f.read()) pose = pose.get_components(["RIGHT_HAND_LANDMARKS"]) @@ -80,7 +80,7 @@ def test_hand_normalization(self): pose.body.data = tensor pose.focus() - with open('data/mediapipe_hand_normalized.pose', 'rb') as f: + with open('tests/data/mediapipe_hand_normalized.pose', 'rb') as f: pose_gold = Pose.read(f.read()) self.assertTrue(ma.allclose(pose.body.data, pose_gold.body.data)) diff --git a/src/python/tests/optical_flow_test.py b/src/python/tests/optical_flow_test.py index 66d0810..d06f636 100644 --- a/src/python/tests/optical_flow_test.py +++ b/src/python/tests/optical_flow_test.py @@ -28,7 +28,7 @@ def test_optical_flow(self): """ calculator = OpticalFlowCalculator(fps=30, distance=DistanceRepresentation()) - with open('data/mediapipe.pose', 'rb') as f: + with open('tests/data/mediapipe.pose', 'rb') as f: pose = Pose.read(f.read()) pose = pose.get_components(["POSE_LANDMARKS", "RIGHT_HAND_LANDMARKS", "LEFT_HAND_LANDMARKS"]) @@ -44,4 +44,4 @@ def test_optical_flow(self): fp = tempfile.NamedTemporaryFile() plt.savefig(fp.name, format='png') - self.assertTrue(compare_images('data/optical_flow.png', fp.name, 0.001) is None) + self.assertTrue(compare_images('tests/data/optical_flow.png', fp.name, 0.001) is None) diff --git a/src/python/tests/visualization_test.py b/src/python/tests/visualization_test.py new file mode 100644 index 0000000..d2e290a --- /dev/null +++ b/src/python/tests/visualization_test.py @@ -0,0 +1,56 @@ +import tempfile +import os +from unittest import TestCase + +from pose_format import Pose +from pose_format.pose_visualizer import PoseVisualizer + + +class TestPoseVisualizer(TestCase): + """ + Test cases for PoseVisualizer functionality. + """ + + def test_save_gif(self): + """ + Test saving pose visualization as GIF. + """ + with open("tests/data/mediapipe.pose", "rb") as f: + pose = Pose.read(f.read()) + + v = PoseVisualizer(pose) + + with tempfile.NamedTemporaryFile(suffix='.gif', delete=False) as temp_gif: + v.save_gif(temp_gif.name, v.draw()) + self.assertTrue(os.path.exists(temp_gif.name)) + self.assertGreater(os.path.getsize( + temp_gif.name), 0) + + def test_save_png(self): + """ + Test saving pose visualization as PNG. + """ + with open("tests/data/mediapipe_long.pose", "rb") as f: + pose = Pose.read(f.read()) + + v = PoseVisualizer(pose) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_png = os.path.join(temp_dir, 'example.png') + v.save_png(temp_png, v.draw(transparency=True)) + self.assertTrue(os.path.exists(temp_png)) + self.assertGreater(os.path.getsize(temp_png), 0) + + def test_save_mp4(self): + """ + Test saving pose visualization as MP4 video. + """ + with open("tests/data/mediapipe_hand_normalized.pose", "rb") as f: + pose = Pose.read(f.read()) + + v = PoseVisualizer(pose) + + with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_mp4: + v.save_video(temp_mp4.name, v.draw()) + self.assertTrue(os.path.exists(temp_mp4.name)) + self.assertGreater(os.path.getsize(temp_mp4.name), 0)