From b976a019c9d0ad04b66415004902d290fd622a2c Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 20:39:20 +1000 Subject: [PATCH 01/17] build: add pack batch script --- pack.bat | 1 + 1 file changed, 1 insertion(+) create mode 100644 pack.bat diff --git a/pack.bat b/pack.bat new file mode 100644 index 0000000..9ce6334 --- /dev/null +++ b/pack.bat @@ -0,0 +1 @@ +python pack.py portal bin \ No newline at end of file From 006b78c2e49cf848bd1d06e6c747b42a62348bf8 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:02:20 +1000 Subject: [PATCH 02/17] feat(color): implement a color class that defines vertex color --- portal/utils/color.py | 80 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 portal/utils/color.py diff --git a/portal/utils/color.py b/portal/utils/color.py new file mode 100644 index 0000000..fb4606d --- /dev/null +++ b/portal/utils/color.py @@ -0,0 +1,80 @@ +from typing import Tuple + +class Color: + def __init__(self, r: int, g: int, b: int, a: float = 1.0) -> None: + """Initialize a Color object with RGB and alpha values.""" + self.r = r + self.g = g + self.b = b + self.a = a + + def to_hex(self) -> str: + """Convert the color to a hexadecimal string.""" + return f"#{self.r:02x}{self.g:02x}{self.b:02x}" + + def to_tuple(self) -> Tuple[int, int, int, float]: + """Return the color as a tuple of (r, g, b, a).""" + return (self.r, self.g, self.b, self.a) + + def to_normalized_tuple(self) -> Tuple[float, float, float, float]: + """Return the color as a normalized tuple of (r, g, b, a).""" + return (self.r / 255, self.g / 255, self.b / 255, self.a) + + def __str__(self) -> str: + """Return the string representation of the color.""" + return f"Color({self.r}, {self.g}, {self.b}, {self.a})" + + +class ColorFactory: + @staticmethod + def from_hex(hex_str: str) -> Color: + """Create a Color object from a hexadecimal string.""" + hex_str = hex_str.lstrip("#") + return Color(*[int(hex_str[i : i + 2], 16) for i in (0, 2, 4)]) + + @staticmethod + def from_rgb(r: int, g: int, b: int, a: float = 1.0) -> Color: + """Create a Color object from RGB and alpha values.""" + return Color(r, g, b, a) + + @staticmethod + def from_tuple(color_tuple: Tuple[int, int, int, float]) -> Color: + """Create a Color object from a tuple of (r, g, b, a).""" + return Color(*color_tuple) + + @staticmethod + def from_normalized_tuple(color_tuple: Tuple[float, float, float, float]) -> Color: + """Create a Color object from a normalized tuple of (r, g, b, a).""" + return Color(*(int(x * 255) for x in color_tuple)) + + +class ColorDecorator: + def __init__(self, color: Color) -> None: + """Initialize a ColorDecorator with a Color object.""" + self._color = color + + def to_hex(self) -> str: + """Convert the decorated color to a hexadecimal string.""" + return self._color.to_hex() + + def to_tuple(self) -> Tuple[int, int, int, float]: + """Return the decorated color as a tuple of (r, g, b, a).""" + return self._color.to_tuple() + + def __str__(self) -> str: + """Return the string representation of the decorated color.""" + return str(self._color) + + +class AlphaColorDecorator(ColorDecorator): + def __init__(self, color: Color, alpha: float) -> None: + """Initialize an AlphaColorDecorator with a Color object and an alpha value.""" + super().__init__(color) + self._color.a = alpha + + +# Example usage: +# color1 = ColorFactory.from_hex("#ff5733") +# color2 = ColorFactory.from_rgb(255, 87, 51) +# decorated_color = AlphaColorDecorator(color1, 0.5) +# print(decorated_color) \ No newline at end of file From ce54d650f20c9f973eed2331f8c17bd28d0b305a Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:02:52 +1000 Subject: [PATCH 03/17] feat(packet): implement a new packet data structure --- portal/data_struct/packet.py | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 portal/data_struct/packet.py diff --git a/portal/data_struct/packet.py b/portal/data_struct/packet.py new file mode 100644 index 0000000..aac05fa --- /dev/null +++ b/portal/data_struct/packet.py @@ -0,0 +1,100 @@ +import struct + + +class PacketHeader: + def __init__(self, is_encrypted, is_compressed, size, checksum): + self.is_compressed = is_compressed + self.is_encrypted = is_encrypted + self.size = size + self.checksum = checksum + + @property + def IsCompressed(self): + return self.is_compressed + + @property + def IsEncrypted(self): + return self.is_encrypted + + @property + def Size(self): + return self.size + + @property + def Checksum(self): + return self.checksum + + @staticmethod + def get_expected_size(): + # see + return 8 # 1 + 1 + 2 + 4 + + +class Packet: + MAGIC_NUMBER = b"pk" # pk + + def __init__( + self, data, size=None, checksum=None, is_encrypted=None, is_compressed=None, header=None + ): + self.data = data + if header is not None: + self.header = header + else: + computed_size = size if size is not None else len(data) + self.header = PacketHeader(is_encrypted, is_compressed, computed_size, checksum) + + def serialize(self): + header_bytes = bytearray() + header_bytes.extend(Packet.MAGIC_NUMBER) # magic number + header_bytes.append(1 if self.header.is_compressed else 0) # is_compressed flag + header_bytes.append(1 if self.header.is_encrypted else 0) # is_encrypted flag + header_bytes.extend(struct.pack("H", self.header.checksum)) # checksum + header_bytes.extend(struct.pack("i", self.header.size)) # size + return bytes(header_bytes) + self.data # combine header and data + + @staticmethod + def validate_magic_number(data): + # minimum size of a packet is the magic number and the header + if len(data) < len(Packet.MAGIC_NUMBER): + raise ValueError("Data is too short to be a valid packet") + + # check magic number + if data[: len(Packet.MAGIC_NUMBER)] != Packet.MAGIC_NUMBER: + raise ValueError("Data does not contain the magic number") + + @staticmethod + def deserialize(data): + Packet.validate_magic_number(data) + index = len(Packet.MAGIC_NUMBER) # start after magic number + header = Packet.deserialize_header(data, index) + + payload_data = data[ + index + PacketHeader.get_expected_size() : index + + PacketHeader.get_expected_size() + + header.Size + ] + packet = Packet(payload_data, header=header) + + return packet + + @staticmethod + def deserialize_header(data, index): + # read flags + is_compressed = data[index] == 1 + index += 1 + is_encrypted = data[index] == 1 + index += 1 + + # read checksum + checksum = struct.unpack_from("H", data, index)[0] + index += struct.calcsize("H") + + # read size + size = struct.unpack_from("i", data, index)[0] + index += struct.calcsize("i") + + return PacketHeader(is_encrypted, is_compressed, size, checksum) + + @staticmethod + def deserialize_header_start(data, start_index=0): + return Packet.deserialize_header(data, start_index) From 6cfa36d1db713e6f8f4edc2b6d9b692d0fb5de99 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:03:23 +1000 Subject: [PATCH 04/17] feat(mmap): handles new packet structure & binary structure --- portal/handlers.py | 73 +++++++++++++++++++++++++++++++----- portal/server/mmap_server.py | 47 +++++++++++------------ 2 files changed, 87 insertions(+), 33 deletions(-) diff --git a/portal/handlers.py b/portal/handlers.py index 52314a7..47deccd 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -1,11 +1,22 @@ import gzip import io import json +import struct +from typing import Tuple import bpy # type: ignore +from .data_struct.packet import PacketHeader +from .utils.color import ColorFactory + class BinaryHandler: + @staticmethod + def parse_header(data: bytes) -> PacketHeader: + # see https://docs.python.org/3/library/struct.html#format-characters + is_compressed, is_encrypted, checksum, size = struct.unpack("??Hi", data) + return PacketHeader(is_encrypted, is_compressed, size, checksum) + @staticmethod def decompress_if_gzip(data: bytes) -> bytes: if data[:2] == b"\x1f\x8b": @@ -19,28 +30,55 @@ def decompress_if_gzip(data: bytes) -> bytes: class DataHandler: @staticmethod - def handle_str_data(data, data_type): + def handle_str_data(payload, data_type): + if payload is None: + return try: if data_type == "Text": - print(f"Received message: {data}") + print(f"Received message: {payload}") elif data_type == "Mesh": - message_dics = json.loads(data) + message_dics = json.loads(payload) for i, item in enumerate(message_dics): - vertices, faces = MeshHandler.deserialize_mesh(item) - MeshHandler.create_or_replace_mesh(f"object_{i}", vertices, faces) + data, metadata = DataHandler.unpack_packet(item) + vertices, faces, colors = MeshHandler.deserialize_mesh(data) + object_name = DataHandler.try_get_name(metadata) + MeshHandler.create_or_replace_mesh( + f"{object_name}_{i}", vertices, faces, colors + ) + except json.JSONDecodeError: + raise ValueError(f"Unsupported data: {payload}") + + def unpack_packet(packet: str) -> Tuple[str, str]: + try: + return packet["Data"], packet["Metadata"] except json.JSONDecodeError: - raise ValueError(f"Unsupported data: {data}") + raise ValueError(f"Unsupported packet data: {packet}") + + def try_get_name(metadata: str) -> str: + try: + return metadata["Name"] + except Exception: + return "object" class MeshHandler: @staticmethod def deserialize_mesh(data): - vertices = [(v["X"], v["Y"], v["Z"]) for v in data["Vertices"]] - faces = [tuple(face_list) for face_list in data["Faces"]] - return vertices, faces + try: + vertices = [(v["X"], v["Y"], v["Z"]) for v in data["Vertices"]] + faces = [tuple(face_list) for face_list in data["Faces"]] + color_hexs = data.get("VertexColors") + if color_hexs and len(color_hexs) == len(vertices): + colors = [ + ColorFactory.from_hex(hex_str).to_normalized_tuple() for hex_str in color_hexs + ] + return vertices, faces, colors + return vertices, faces, None + except KeyError: + raise ValueError(f"Unsupported mesh data structure: {data}") @staticmethod - def create_or_replace_mesh(object_name, vertices, faces): + def create_or_replace_mesh(object_name, vertices, faces, vertex_colors=None): obj = bpy.data.objects.get(object_name) new_mesh_data = bpy.data.meshes.new(f"{object_name}_mesh") new_mesh_data.from_pydata(vertices, [], faces) @@ -54,4 +92,19 @@ def create_or_replace_mesh(object_name, vertices, faces): new_object = bpy.data.objects.new(object_name, new_mesh_data) bpy.context.collection.objects.link(new_object) + # Assign vertex colors if provided + if vertex_colors: + if not new_mesh_data.vertex_colors: + new_mesh_data.vertex_colors.new() + + color_layer = new_mesh_data.vertex_colors.active + color_dict = {i: col for i, col in enumerate(vertex_colors)} + + for poly in new_mesh_data.polygons: + for idx in poly.loop_indices: + loop = new_mesh_data.loops[idx] + vertex_index = loop.vertex_index + if vertex_index in color_dict: + color_layer.data[idx].color = color_dict[vertex_index] + new_mesh_data.update() diff --git a/portal/server/mmap_server.py b/portal/server/mmap_server.py index 1ecbd58..33c0915 100644 --- a/portal/server/mmap_server.py +++ b/portal/server/mmap_server.py @@ -1,11 +1,11 @@ import mmap import queue -import struct import threading import time import bpy # type: ignore +from ..data_struct.packet import Packet from ..handlers import BinaryHandler @@ -13,7 +13,7 @@ class MMFServerManager: data_queue = queue.Queue() shutdown_event = threading.Event() _server_thread = None - _last_hash = None + _last_checksum = None mmf = None @staticmethod @@ -21,31 +21,32 @@ def handle_raw_bytes(mmf): try: while not MMFServerManager.shutdown_event.is_set(): mmf.seek(0) - if mmf.size() >= 20: - hash_prefix = mmf.read(16) # md5 hash is 16 bytes - - # Check if the data is the same as the last read - if hash_prefix != MMFServerManager._last_hash: - MMFServerManager._last_hash = hash_prefix - length_prefix = mmf.read(4) - data_length = struct.unpack("I", length_prefix)[0] - # print(f"Data length: {data_length}") - - if data_length > 0 and mmf.size() >= 4 + data_length: - data = mmf.read(data_length) - data = BinaryHandler.decompress_if_gzip(data) - MMFServerManager.data_queue.put(data.decode("utf-8")) - else: - raise ValueError("Data length exceeds the current readable size.") - else: - # print("Data is the same as the last read.") - pass + if mmf.size() >= 10: + Packet.validate_magic_number(mmf.read(2)) + header = BinaryHandler.parse_header(mmf.read(8)) + checksum = header.Checksum + # Only process data if checksum is different from the last one + if checksum != MMFServerManager._last_checksum: + print( + f"isCompressed: {header.IsCompressed}, isEncrypted: {header.IsEncrypted}, checksum: {checksum}, size: {header.Size}" + ) + MMFServerManager._last_checksum = checksum + data = mmf.read(header.Size) + data = BinaryHandler.decompress_if_gzip(data) + try: + decoded_data = data.decode("utf-8") + except UnicodeDecodeError: + raise ValueError("Received data cannot be decoded as UTF-8.") + MMFServerManager.data_queue.put(decoded_data) + time.sleep(bpy.context.scene.event_timer) else: raise ValueError( "Not enough data to read hash & length prefix. " - + "Data should follows the format: '[16b byte[] hash] [4b int32 length] [data]'" + + "Packet should follow the format: \n" + + "'[2b byte[] magic_num] [1b bool isCompressed] [1b bool isEncrypted] [2b int16 checksum] [4b int32 size] [payload]'" ) - time.sleep(bpy.context.scene.event_timer) # Adjust as needed + except ValueError as ve: + raise RuntimeError(f"Value Error in handle_mmf_data: {ve}") except Exception as e: raise RuntimeError(f"Error in handle_mmf_data: {e}") From b845c453ee7b40b756534b0d4bdb026530abf930 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:03:39 +1000 Subject: [PATCH 05/17] update .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 24aceb2..0786565 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.zip /bin -venv/ \ No newline at end of file +venv/ +__pycache__/ +*.pyc \ No newline at end of file From b8985fe31545327f97b9d2c842b3af7ab64b7526 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:24:24 +1000 Subject: [PATCH 06/17] feat(server): implement new data structure for all server --- portal/server/pipe_server.py | 16 +++++++++------- portal/server/udp_server.py | 4 +++- portal/server/websockets_server.py | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/portal/server/pipe_server.py b/portal/server/pipe_server.py index a68ea71..f63557e 100644 --- a/portal/server/pipe_server.py +++ b/portal/server/pipe_server.py @@ -1,10 +1,10 @@ import queue -import struct import threading import time import bpy # type: ignore +from ..data_struct.packet import Packet, PacketHeader from ..handlers import BinaryHandler # Attempt to import the pywin32 modules safely @@ -33,12 +33,14 @@ def handle_raw_bytes(pipe): try: while not PipeServerManager.shutdown_event.is_set(): try: - size_prefix = win32file.ReadFile(pipe, 4, None)[1] - (size,) = struct.unpack("I", size_prefix) - if size == 0: - break - - data = win32file.ReadFile(pipe, size, None)[1] + signature = win32file.ReadFile(pipe, 2, None)[1] + Packet.validate_magic_number(signature) + header_bytes = win32file.ReadFile(pipe, PacketHeader.get_expected_size(), None)[ + 1 + ] + header = BinaryHandler.parse_header(header_bytes) + + data = win32file.ReadFile(pipe, header.size, None)[1] data = BinaryHandler.decompress_if_gzip(data).decode("utf-8") PipeServerManager.data_queue.put(data) except pywintypes.error as e: diff --git a/portal/server/udp_server.py b/portal/server/udp_server.py index 89799cb..af8777e 100644 --- a/portal/server/udp_server.py +++ b/portal/server/udp_server.py @@ -19,7 +19,9 @@ def udp_handler(): try: # 1500 is the max size of a UDP packet data, addr = UDPServerManager._sock.recvfrom(1500) - data = BinaryHandler.decompress_if_gzip(data) + header = BinaryHandler.parse_header(data) + payload = data[header.get_expected_size() + 2 :] + data = BinaryHandler.decompress_if_gzip(payload) UDPServerManager.data_queue.put(data.decode("utf-8")) except socket.timeout: continue diff --git a/portal/server/websockets_server.py b/portal/server/websockets_server.py index 01cbfe9..2a93a2c 100644 --- a/portal/server/websockets_server.py +++ b/portal/server/websockets_server.py @@ -35,7 +35,9 @@ async def websocket_handler(request): async for msg in ws: if msg.type == aiohttp.WSMsgType.BINARY: data = msg.data - data = BinaryHandler.decompress_if_gzip(data) + header = BinaryHandler.parse_header(data) + payload = data[header.get_expected_size() + 2 :] + data = BinaryHandler.decompress_if_gzip(payload) WebSocketServerManager.data_queue.put(data.decode("utf-8")) elif msg.type == aiohttp.WSMsgType.ERROR: raise RuntimeError( From 834ad63aeb556fb0b531c9c3cb2096990ddf95ff Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sat, 31 Aug 2024 23:26:12 +1000 Subject: [PATCH 07/17] refactor: use variable for binary parsing instead of hard-coded sizes for safety --- portal/handlers.py | 4 +++- portal/server/mmap_server.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/portal/handlers.py b/portal/handlers.py index 47deccd..0e56b38 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -14,7 +14,9 @@ class BinaryHandler: @staticmethod def parse_header(data: bytes) -> PacketHeader: # see https://docs.python.org/3/library/struct.html#format-characters - is_compressed, is_encrypted, checksum, size = struct.unpack("??Hi", data) + is_compressed, is_encrypted, checksum, size = struct.unpack( + "??Hi", data[: PacketHeader.get_expected_size()] + ) return PacketHeader(is_encrypted, is_compressed, size, checksum) @staticmethod diff --git a/portal/server/mmap_server.py b/portal/server/mmap_server.py index 33c0915..4893ef9 100644 --- a/portal/server/mmap_server.py +++ b/portal/server/mmap_server.py @@ -5,7 +5,7 @@ import bpy # type: ignore -from ..data_struct.packet import Packet +from ..data_struct.packet import Packet, PacketHeader from ..handlers import BinaryHandler @@ -23,7 +23,7 @@ def handle_raw_bytes(mmf): mmf.seek(0) if mmf.size() >= 10: Packet.validate_magic_number(mmf.read(2)) - header = BinaryHandler.parse_header(mmf.read(8)) + header = BinaryHandler.parse_header(mmf.read(PacketHeader.get_expected_size())) checksum = header.Checksum # Only process data if checksum is different from the last one if checksum != MMFServerManager._last_checksum: From ab7e8aa9ed7c1cd00d00d13228a97ba2f2ade7f3 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 00:43:57 +1000 Subject: [PATCH 08/17] feat: add handler for material metadata --- portal/handlers.py | 58 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/portal/handlers.py b/portal/handlers.py index 0e56b38..de339e1 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -44,8 +44,9 @@ def handle_str_data(payload, data_type): data, metadata = DataHandler.unpack_packet(item) vertices, faces, colors = MeshHandler.deserialize_mesh(data) object_name = DataHandler.try_get_name(metadata) + material = DataHandler.try_get_material(metadata) MeshHandler.create_or_replace_mesh( - f"{object_name}_{i}", vertices, faces, colors + f"{object_name}_{i}", vertices, faces, colors, material ) except json.JSONDecodeError: raise ValueError(f"Unsupported data: {payload}") @@ -62,6 +63,12 @@ def try_get_name(metadata: str) -> str: except Exception: return "object" + def try_get_material(metadata: str) -> str | None: + try: + return metadata["Material"] + except Exception: + return None + class MeshHandler: @staticmethod @@ -80,7 +87,7 @@ def deserialize_mesh(data): raise ValueError(f"Unsupported mesh data structure: {data}") @staticmethod - def create_or_replace_mesh(object_name, vertices, faces, vertex_colors=None): + def create_or_replace_mesh(object_name, vertices, faces, vertex_colors=None, material=None): obj = bpy.data.objects.get(object_name) new_mesh_data = bpy.data.meshes.new(f"{object_name}_mesh") new_mesh_data.from_pydata(vertices, [], faces) @@ -93,20 +100,45 @@ def create_or_replace_mesh(object_name, vertices, faces, vertex_colors=None): else: new_object = bpy.data.objects.new(object_name, new_mesh_data) bpy.context.collection.objects.link(new_object) + obj = new_object # Assign vertex colors if provided if vertex_colors: - if not new_mesh_data.vertex_colors: - new_mesh_data.vertex_colors.new() + MeshHandler.apply_vertex_colors(new_mesh_data, vertex_colors) - color_layer = new_mesh_data.vertex_colors.active - color_dict = {i: col for i, col in enumerate(vertex_colors)} - - for poly in new_mesh_data.polygons: - for idx in poly.loop_indices: - loop = new_mesh_data.loops[idx] - vertex_index = loop.vertex_index - if vertex_index in color_dict: - color_layer.data[idx].color = color_dict[vertex_index] + # Assign material if provided + if material: + print(f"Applying material '{material}' to object '{object_name}'") + MeshHandler.apply_material(obj, material) new_mesh_data.update() + + @staticmethod + def apply_vertex_colors(mesh_data, vertex_colors): + if not mesh_data.vertex_colors: + mesh_data.vertex_colors.new() + + color_layer = mesh_data.vertex_colors.active + color_dict = {i: col for i, col in enumerate(vertex_colors)} + + for poly in mesh_data.polygons: + for idx in poly.loop_indices: + loop = mesh_data.loops[idx] + vertex_index = loop.vertex_index + if vertex_index in color_dict: + color_layer.data[idx].color = color_dict[vertex_index] + + @staticmethod + def apply_material(obj, material): + mat = bpy.data.materials.get(material) + if mat: + # if the object has no material slots, add one + if len(obj.data.materials) == 0: + obj.data.materials.append(mat) + else: + obj.data.materials[0] = mat + + return True + else: + print(f"Material {material} not found.") + return False From 717e6d87bbc2bfffd1c430a60f5e8ac01a172c7b Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 00:44:16 +1000 Subject: [PATCH 09/17] update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0786565..008aadb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /bin venv/ __pycache__/ -*.pyc \ No newline at end of file +*.pyc +test_project[s] \ No newline at end of file From 65d3fd34bb7bb39a001d13c76f8ed8d2d042bcc5 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 00:45:10 +1000 Subject: [PATCH 10/17] refactor: remove debug print --- portal/handlers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/portal/handlers.py b/portal/handlers.py index de339e1..762fc9f 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -108,7 +108,6 @@ def create_or_replace_mesh(object_name, vertices, faces, vertex_colors=None, mat # Assign material if provided if material: - print(f"Applying material '{material}' to object '{object_name}'") MeshHandler.apply_material(obj, material) new_mesh_data.update() From fd3ed35b77aadce31f1a2cbbd62659d16e0236e3 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 01:07:27 +1000 Subject: [PATCH 11/17] feat: use header to determine if compressed and decompress accordingly --- portal/handlers.py | 8 ++++---- portal/server/mmap_server.py | 3 ++- portal/server/pipe_server.py | 5 +++-- portal/server/udp_server.py | 3 ++- portal/server/websockets_server.py | 3 ++- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/portal/handlers.py b/portal/handlers.py index 762fc9f..3ee8d6f 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -20,14 +20,14 @@ def parse_header(data: bytes) -> PacketHeader: return PacketHeader(is_encrypted, is_compressed, size, checksum) @staticmethod - def decompress_if_gzip(data: bytes) -> bytes: - if data[:2] == b"\x1f\x8b": - with gzip.GzipFile(fileobj=io.BytesIO(data)) as gz: + def decompress(data: bytes) -> bytes: + if not data[:2] == b"\x1f\x8b": + raise ValueError("Data is not in gzip format.") + with gzip.GzipFile(fileobj=io.BytesIO(data)) as gz: try: return gz.read() except OSError: return data - return data class DataHandler: diff --git a/portal/server/mmap_server.py b/portal/server/mmap_server.py index 4893ef9..6d201e1 100644 --- a/portal/server/mmap_server.py +++ b/portal/server/mmap_server.py @@ -32,7 +32,8 @@ def handle_raw_bytes(mmf): ) MMFServerManager._last_checksum = checksum data = mmf.read(header.Size) - data = BinaryHandler.decompress_if_gzip(data) + if header.IsCompressed: + data = BinaryHandler.decompress(data) try: decoded_data = data.decode("utf-8") except UnicodeDecodeError: diff --git a/portal/server/pipe_server.py b/portal/server/pipe_server.py index f63557e..88001b5 100644 --- a/portal/server/pipe_server.py +++ b/portal/server/pipe_server.py @@ -41,8 +41,9 @@ def handle_raw_bytes(pipe): header = BinaryHandler.parse_header(header_bytes) data = win32file.ReadFile(pipe, header.size, None)[1] - data = BinaryHandler.decompress_if_gzip(data).decode("utf-8") - PipeServerManager.data_queue.put(data) + if header.is_compressed: + data = BinaryHandler.decompress(data) + PipeServerManager.data_queue.put(data.decode("utf-8")) except pywintypes.error as e: if e.winerror == 109: # ERROR_BROKEN_PIPE break diff --git a/portal/server/udp_server.py b/portal/server/udp_server.py index af8777e..fcc3626 100644 --- a/portal/server/udp_server.py +++ b/portal/server/udp_server.py @@ -21,7 +21,8 @@ def udp_handler(): data, addr = UDPServerManager._sock.recvfrom(1500) header = BinaryHandler.parse_header(data) payload = data[header.get_expected_size() + 2 :] - data = BinaryHandler.decompress_if_gzip(payload) + if header.is_compressed: + data = BinaryHandler.decompress(payload) UDPServerManager.data_queue.put(data.decode("utf-8")) except socket.timeout: continue diff --git a/portal/server/websockets_server.py b/portal/server/websockets_server.py index 2a93a2c..d561724 100644 --- a/portal/server/websockets_server.py +++ b/portal/server/websockets_server.py @@ -37,7 +37,8 @@ async def websocket_handler(request): data = msg.data header = BinaryHandler.parse_header(data) payload = data[header.get_expected_size() + 2 :] - data = BinaryHandler.decompress_if_gzip(payload) + if header.is_compressed: + data = BinaryHandler.decompress(payload) WebSocketServerManager.data_queue.put(data.decode("utf-8")) elif msg.type == aiohttp.WSMsgType.ERROR: raise RuntimeError( From 5c8ab130302ffdbd2485c8aaa187b95441f184fb Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 02:01:08 +1000 Subject: [PATCH 12/17] refactor: format code indentation --- portal/handlers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portal/handlers.py b/portal/handlers.py index 3ee8d6f..9def55e 100644 --- a/portal/handlers.py +++ b/portal/handlers.py @@ -24,10 +24,10 @@ def decompress(data: bytes) -> bytes: if not data[:2] == b"\x1f\x8b": raise ValueError("Data is not in gzip format.") with gzip.GzipFile(fileobj=io.BytesIO(data)) as gz: - try: - return gz.read() - except OSError: - return data + try: + return gz.read() + except OSError: + return data class DataHandler: From 3be9688915e483fb72efe472b24d7dae3cb7958e Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 02:02:44 +1000 Subject: [PATCH 13/17] feat(security): add encryption check with not implemented error --- portal/server/mmap_server.py | 2 ++ portal/server/pipe_server.py | 2 ++ portal/server/udp_server.py | 2 ++ portal/server/websockets_server.py | 2 ++ 4 files changed, 8 insertions(+) diff --git a/portal/server/mmap_server.py b/portal/server/mmap_server.py index 6d201e1..0d69029 100644 --- a/portal/server/mmap_server.py +++ b/portal/server/mmap_server.py @@ -34,6 +34,8 @@ def handle_raw_bytes(mmf): data = mmf.read(header.Size) if header.IsCompressed: data = BinaryHandler.decompress(data) + if header.IsEncrypted: + raise NotImplementedError("Encrypted data is not supported.") try: decoded_data = data.decode("utf-8") except UnicodeDecodeError: diff --git a/portal/server/pipe_server.py b/portal/server/pipe_server.py index 88001b5..f7cc7f0 100644 --- a/portal/server/pipe_server.py +++ b/portal/server/pipe_server.py @@ -43,6 +43,8 @@ def handle_raw_bytes(pipe): data = win32file.ReadFile(pipe, header.size, None)[1] if header.is_compressed: data = BinaryHandler.decompress(data) + if header.is_encrypted: + raise NotImplementedError("Encrypted data is not supported.") PipeServerManager.data_queue.put(data.decode("utf-8")) except pywintypes.error as e: if e.winerror == 109: # ERROR_BROKEN_PIPE diff --git a/portal/server/udp_server.py b/portal/server/udp_server.py index fcc3626..1f923b7 100644 --- a/portal/server/udp_server.py +++ b/portal/server/udp_server.py @@ -23,6 +23,8 @@ def udp_handler(): payload = data[header.get_expected_size() + 2 :] if header.is_compressed: data = BinaryHandler.decompress(payload) + if header.is_encrypted: + raise NotImplementedError("Encrypted data is not supported.") UDPServerManager.data_queue.put(data.decode("utf-8")) except socket.timeout: continue diff --git a/portal/server/websockets_server.py b/portal/server/websockets_server.py index d561724..a7fb765 100644 --- a/portal/server/websockets_server.py +++ b/portal/server/websockets_server.py @@ -39,6 +39,8 @@ async def websocket_handler(request): payload = data[header.get_expected_size() + 2 :] if header.is_compressed: data = BinaryHandler.decompress(payload) + if header.is_encrypted: + raise NotImplementedError("Encrypted data is not supported.") WebSocketServerManager.data_queue.put(data.decode("utf-8")) elif msg.type == aiohttp.WSMsgType.ERROR: raise RuntimeError( From c66ef6e2af32e852799c673bd0e8473146edc7b3 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 02:11:31 +1000 Subject: [PATCH 14/17] feat(ui): move event_timer to global property rather than connection specific property --- portal/panels.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/portal/panels.py b/portal/panels.py index aedd11d..1d18ba5 100644 --- a/portal/panels.py +++ b/portal/panels.py @@ -75,6 +75,7 @@ def draw(self, context): # Data type selection layout.prop(scene, "data_type") + layout.prop(scene, "event_timer") # Start/Stop server buttons row = layout.row(align=True) @@ -96,32 +97,31 @@ def draw_status(self, layout, connection_type): def draw_named_pipe(self, layout, scene): col = layout.column() col.prop(scene, "pipe_name") - col.prop(scene, "event_timer") def draw_mmap(self, layout, scene): col = layout.column() col.prop(scene, "mmf_name") col.prop(scene, "buffer_size") - col.prop(scene, "event_timer") def draw_websockets(self, layout, scene): col = layout.column() col.prop(scene, "port") col.prop(scene, "route") - col.prop(scene, "event_timer") col.prop(scene, "is_external") def draw_udp(self, layout, scene): col = layout.column() col.prop(scene, "port") - col.prop(scene, "event_timer") col.prop(scene, "is_external") def register_properties(): # default initial connection type register_connection_properties("NAMED_PIPE") - + safe_register_property( + "event_timer", + bpy.props.FloatProperty(name="Interval (sec)", default=0.01, min=0.001, max=1.0), + ) safe_register_property( "data_type", bpy.props.EnumProperty( @@ -137,6 +137,7 @@ def register_properties(): def unregister_properties(): safe_unregister_property("data_type") + safe_unregister_property("event_timer") def register_connection_properties(connection_type): @@ -144,10 +145,6 @@ def register_connection_properties(connection_type): safe_register_property( "pipe_name", bpy.props.StringProperty(name="Name", default="testpipe") ) - safe_register_property( - "event_timer", - bpy.props.FloatProperty(name="Interval (sec)", default=0.01, min=0.001, max=1.0), - ) elif connection_type == "MMAP": safe_register_property( "mmf_name", bpy.props.StringProperty(name="Name", default="memory_file") @@ -155,17 +152,9 @@ def register_connection_properties(connection_type): safe_register_property( "buffer_size", bpy.props.IntProperty(name="Buffer Size (KB)", default=1024) ) - safe_register_property( - "event_timer", - bpy.props.FloatProperty(name="Interval (sec)", default=0.01, min=0.001, max=1.0), - ) elif connection_type == "WEBSOCKETS": safe_register_property("port", bpy.props.IntProperty(name="Port", default=8765)) safe_register_property("route", bpy.props.StringProperty(name="route", default="/")) - safe_register_property( - "event_timer", - bpy.props.FloatProperty(name="Interval (sec)", default=0.01, min=0.001, max=1.0), - ) safe_register_property( "is_external", bpy.props.BoolProperty( @@ -176,10 +165,6 @@ def register_connection_properties(connection_type): ) elif connection_type == "UDP": safe_register_property("port", bpy.props.IntProperty(name="Port", default=8765)) - safe_register_property( - "event_timer", - bpy.props.FloatProperty(name="Interval (sec)", default=0.01, min=0.001, max=1.0), - ) safe_register_property( "is_external", bpy.props.BoolProperty( @@ -193,7 +178,6 @@ def register_connection_properties(connection_type): def unregister_connection_properties(scene): props_to_remove = [ "pipe_name", - "event_timer", "mmf_name", "port", "route", From 069d1c7c13de47ebcdb1daa35e16b6941c07811c Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 03:25:10 +1000 Subject: [PATCH 15/17] bump version to `0.0.3` --- portal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portal/__init__.py b/portal/__init__.py index 4a32541..cc2c8db 100644 --- a/portal/__init__.py +++ b/portal/__init__.py @@ -12,7 +12,7 @@ bl_info = { "name": "Portal", "author": "Zeke Zhang", - "version": (0, 0, 2), + "version": (0, 0, 3), "blender": (4, 2, 0), "category": "System", "location": "View3D > Sidebar > Portal", From 2d17a356a993c59d91efecbf2fba9b2cd83f509f Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 03:25:20 +1000 Subject: [PATCH 16/17] update .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 008aadb..e9b1e49 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ venv/ __pycache__/ *.pyc -test_project[s] \ No newline at end of file +test_project[s] +dist/ +build/ \ No newline at end of file From 2c9c8dfcab64d5b48cadca72cd755a4bbaee4692 Mon Sep 17 00:00:00 2001 From: Zeke Zhang Date: Sun, 1 Sep 2024 03:26:23 +1000 Subject: [PATCH 17/17] build: add compile script for Blender plugin into single Python file --- compile.py | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 compile.py diff --git a/compile.py b/compile.py new file mode 100644 index 0000000..f784d58 --- /dev/null +++ b/compile.py @@ -0,0 +1,150 @@ +import ast +import os +import pathlib +import re + + +def merge_files(output_file, source_files): + import_set = set() # To keep track of unique imports + merged_code = "" + + for fname in source_files: + with open(fname) as infile: + for line in infile: + # Check if the line is an import statement + import_match = re.match(r"^\s*(import|from)\s+(\S+)", line) + if import_match: + import_statement = import_match.group(0) + # Skip local imports that start with a dot + if not re.match(r"^\s*(import|from)\s+\.", import_statement): + # Add to import set if it's a new import + if import_statement not in import_set: + import_set.add(import_statement) + merged_code += line + else: + # Add non-import lines to the merged code + merged_code += line + merged_code += "\n\n" # Ensure separation between files + + # create output directory if it doesn't exist + os.makedirs(os.path.dirname(output_file), exist_ok=True) + # Write the merged code to the output file + with open(output_file, "w") as outfile: + outfile.write(merged_code) + + print(f"Merged files into {output_file}") + + +# ------------------------------------------------------------ +# Post processing +# ------------------------------------------------------------ + + +def list_top_level_functions(source_file): + with open(source_file, "r") as file: + tree = ast.parse(file.read(), filename=source_file) + + attach_parents(tree) # Attach parent references before processing + + top_level_functions = [] + + for node in ast.walk(tree): + # Check for top-level function definitions + if isinstance(node, ast.FunctionDef): + # Ensure the function is not within a class + if not within_class(node): + top_level_functions.append(node) + + return top_level_functions + + +def within_class(node): + """ + Check if the given node is within a class definition. + """ + while hasattr(node, "parent"): + node = node.parent + if isinstance(node, ast.ClassDef): + return True + return False + + +def attach_parents(tree): + """ + Attach parent references to each node in the AST. + """ + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + + +def remove_duplicate_functions(source_file): + with open(source_file, "r") as file: + tree = ast.parse(file.read(), filename=source_file) + + attach_parents(tree) # Attach parent references before processing + + func_names = set() + nodes_to_remove = [] + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + if not within_class(node): + if node.name in func_names: + nodes_to_remove.append(node) + else: + func_names.add(node.name) + + # Remove the nodes from the source by reconstructing the source code + lines = open(source_file).readlines() + for node in nodes_to_remove: + start_lineno = node.lineno - 1 + end_lineno = node.end_lineno if hasattr(node, "end_lineno") else node.lineno + for i in range(start_lineno, end_lineno): + lines[i] = "" + + with open(source_file, "w") as file: + file.writelines(lines) + + print(f"Removed duplicate top-level functions from {source_file}") + + +def read_version(source_dir): + version_file = pathlib.Path(source_dir, "__init__.py") + if version_file.exists(): + with version_file.open() as f: + content = f.read() + version_match = re.search(r"\"version\":\s*\((\d+),\s*(\d+),\s*(\d+)\)", content) + if version_match: + version = ".".join(version_match.groups()) + return version + return None + + +if __name__ == "__main__": + # List your Python files in the order you want them to be merged. + source_files = [ + "portal/utils/color.py", + "portal/data_struct/packet.py", + "portal/handlers.py", + "portal/server/mmap_server.py", + "portal/server/udp_server.py", + "portal/server/pipe_server.py", + "portal/server/websockets_server.py", + "portal/managers.py", + "portal/operators.py", + "portal/panels.py", + "portal/__init__.py", + ] + + # Output single script file + version = read_version("portal") + output_file = f"bin/portal.blender-{version}.py" + + merge_files(output_file, source_files) + + remove_duplicate_functions(output_file) + + with open(output_file, "r") as file: + tree = ast.parse(file.read(), filename=output_file) + attach_parents(tree) \ No newline at end of file