sounder: fixed devices when reading .mid files

tools/sci/sounder/midi.py

@@ -1,13 +1,20 @@
 import math
 import threading
 import time
+import re
 from copy import deepcopy
+from ast import literal_eval
 
 import mido
 from mido import MidiFile, MidiTrack
 from mido.midifiles.tracks import _to_abstime, _to_reltime
 
 from utils import logger, get_all_channels
+from sci_common import SCI1_Devices, ChannelInfo
+
+
+def is_regular_msg(m):
+    return not m.is_realtime and not m.is_meta and m.type != 'sysex'
 
 
 def get_midi_channels_of_device(play_device, devices, ):
@@ -73,6 +80,34 @@ def play_midi(midi_wave, play_device, port=None, verbose=False, gooey_enabled=Fa
         port.send(msg)
 
 
+def get_midi_devices(midifile):
+    devices = {}
+    # get devices information from midi information track (if exists - probably created by us, when reading a SCI0 file)
+    for msg in midifile.tracks[0]:
+        if msg.type == 'device_name' and msg.name.startswith('Device '):
+            m = re.match(r'Device (.*) uses (\[.*)', msg.name)
+            if m:
+                try:
+                    device = SCI1_Devices[m.group(1)]
+                    channels = literal_eval(m.group(2))
+                    devices[device] = channels
+                except KeyError:
+                    logger.info(f"SAVE SCI1+: Ignoring device {m.group(1)}, doesn't have a SCI1 counterpart")
+
+    if not devices:
+        # make devices table, if devices information not found in midi file
+        channel_nums = sorted(list(set([m.channel for m in mido.merge_tracks(midifile.tracks) if is_regular_msg(m)])))
+        if channel_nums:
+            logger.info(
+                "Couldn't find devices information in first track; using arbitrary values. Contact Zvika if you wish to have control over this")
+            channels = [ChannelInfo(num=ch) for ch in channel_nums]
+            devices[SCI1_Devices.GM] = channels
+            devices[SCI1_Devices.ADLIB] = channels
+            devices[SCI1_Devices.SPEAKER] = [ChannelInfo(num=channel_nums[0])]
+
+    return devices
+
+
 def read_midi_file(p):
     midifile = MidiFile(p)
     for track in midifile.tracks:
@@ -83,7 +118,7 @@ def read_midi_file(p):
                 except:
                     pass
 
-    return midifile
+    return {'midifile': midifile, 'wave': None, 'devices': get_midi_devices(midifile)}
 
 
 def save_midi(midi_wave, input_file, save_file, save_midi_device):

tools/sci/sounder/sci0.py

@@ -208,7 +208,7 @@ def find_last_non_digital_offset(stream):
 def clean_stops(messages):
     # required for SCI0, if there is digital channel - it's identified by looking for 0xFC (or 2)
     # but we might have more 0xFC-s in our file, if each track had it's own 'stop' command
-    # this leaves only the last STOP (0xFC) command
+    # this leaves only the last STOP (0xFC) command ...
     result = []
     redundant_stops = len([m for m in messages if m.type == 'stop']) - 1  # remove all but one
     stops = 0
@@ -219,5 +219,8 @@ def clean_stops(messages):
             stops += 1
             if stops > redundant_stops:
                 result.append(msg)
+    # ... and this adds an ending stop if non existed
+    if stops == 0:
+        result.append(mido.Message(type='stop'))
     assert len([m for m in result if m.type == 'stop']) == 1
     return result

tools/sci/sounder/sci1.py

@@ -1,13 +1,12 @@
 import io
-import re
-from ast import literal_eval
 
 import mido
 from mido import MidiFile, MidiTrack
 
 from sci_common import SIERRA_SND_HEADER, TICKS_PER_BIT, SCI1_Devices, ChannelInfo
 from sci_common import get_sierra_delay_bytes, read_messages
 from utils import read_le, logger, write_le
+from midi import is_regular_msg
 
 SCI1_DIGITAL_CHANNEL_MARKER = 0xfe
 
@@ -115,20 +114,6 @@ def save_sci1(midi_wave, input_file, save_file):
         except KeyError:
             logger.info(f"SAVE SCI1: Ignoring device {orig_device.name}, doesn't have a SCI1 counterpart")
 
-    # TODO: move this out of here
-    # get devices information from midi information track (if exists - probably created by us, when reading a SCI0 file)
-    if not devices:
-        for msg in midifile.tracks[0]:
-            if msg.type == 'device_name' and msg.name.startswith('Device '):
-                m = re.match(r'Device (.*) uses (\[.*)', msg.name)
-                if m:
-                    try:
-                        device = SCI1_Devices[m.group(1)]
-                        channels = literal_eval(m.group(2))
-                        devices[device] = channels
-                    except KeyError:
-                        logger.info(f"SAVE SCI1+: Ignoring device {m.group(1)}, doesn't have a SCI1 counterpart")
-
     # SCI1 makes heavy use of GM - add such track if doesn't exist
     if SCI1_Devices.GM not in devices and SCI1_Devices.MT_32 in devices:
         devices[SCI1_Devices.GM] = devices[SCI1_Devices.MT_32]
@@ -145,7 +130,7 @@ def save_sci1(midi_wave, input_file, save_file):
     # prepare channels data, will be written to file later
     channel_offsets = {}
     channel_sizes = {}
-    channel_nums = sorted(list(set([m.channel for m in messages if not m.is_realtime and not m.is_meta])))
+    channel_nums = sorted(list(set([m.channel for m in messages if is_regular_msg(m)])))
     if midi_wave['wave'] and SCI1_DIGITAL_CHANNEL_MARKER not in channel_nums:
         channel_nums.append(SCI1_DIGITAL_CHANNEL_MARKER)
 
@@ -155,7 +140,7 @@ def save_sci1(midi_wave, input_file, save_file):
             channel_messages[ch] = []
             timer = 0
             for msg in messages:
-                if not msg.is_meta and (msg.is_realtime or msg.channel == ch):
+                if not msg.is_meta and msg.type != 'sysex' and (msg.is_realtime or msg.channel == ch):
                     delta = msg.time - timer
                     timer = msg.time
                     m = msg.copy()
@@ -191,15 +176,6 @@ def save_sci1(midi_wave, input_file, save_file):
         if len(channels_bytes) > 0xffff:
             logger.warning(f"File too big. It's {len(channels_bytes)} bytes, while maximal size should be 65536 bytes")
 
-    # make devices table, if doesn't exists
-    if channel_nums and not devices:
-        logger.info(
-            "Couldn't find devices information in first track; using arbitrary values. Contact Zvika if you wish to have control over this")
-        channels = [ChannelInfo(num=ch) for ch in channel_nums]
-        devices[SCI1_Devices.GM] = channels
-        devices[SCI1_Devices.ADLIB] = channels
-        devices[SCI1_Devices.SPEAKER] = [ChannelInfo(num=channel_nums[0])]
-
     # sq3, sound.016, some devices contains non existing channels - clean them
     for device in devices:
         for channel_info in devices[device]:

tools/sci/sounder/sounder.py

@@ -1,6 +1,3 @@
-# TODO: add .mid files to unit tests
-# TODO: midi: read write devices
-
 # TODO: license
 # TODO: github: issues - templates?
 
@@ -26,13 +23,13 @@
 
 import gooey_misc
 from digital import play_wave, save_wave, read_wav_file, convert_audio_to_low_wav
-from midi import play_midi, read_midi_file, save_midi
+from midi import play_midi, read_midi_file, save_midi, is_regular_msg
 from sci_common import SIERRA_SND_HEADER, SCI0_Early_Devices, SCI0_Devices, SCI1_Devices
 from sci0 import read_sci0_snd_file, save_sci0
 from sci1 import read_sci1_snd_file, save_sci1
 from utils import logger
 
-VERSION = "0.4"
+VERSION = "0.5"
 
 
 def read_snd_file(p, input_version, info):
@@ -69,15 +66,14 @@ def read_input(input_file, input_version, input_digital, info):
     if not p.exists():
         raise FileExistsError(f"File doesn't exist: {p.absolute()}")
     if p.suffix.lower() == '.mid':
-        midifile = read_midi_file(p)
-        result = {'midifile': midifile, 'wave': None}
+        result = read_midi_file(p)
     elif p.suffix.lower() == '.snd' or p.stem.lower().startswith('sound'):
         result = read_snd_file(p, input_version, info)
     else:
         raise NameError("Received unsupported file (it should start with sound. or end with .mid/.snd) " + input_file)
     if info:
         messages = mido.merge_tracks(result['midifile'].tracks)
-        channel_nums = sorted(list(set([m.channel for m in messages if not m.is_realtime and not m.is_meta])))
+        channel_nums = sorted(list(set([m.channel for m in messages if is_regular_msg(m)])))
         logger.debug(f"Channels actually used in messages: {[c + 1 for c in channel_nums]}")
         logger.info(f"Midi length: {result['midifile'].length:.1f} seconds")
 

tools/sci/sounder/tests/test_sounder.py

@@ -1,8 +1,10 @@
 from unittest import TestCase
 
+import mido
+
 from tools.sci.sounder.sounder import *
 
-# in order to run, please put relevant sound.* or *.snd file under 'sound files/*/*/'
+# in order to run, please put relevant sound.*, *.snd and *.mid files under 'sound files/{sci0,sci0_early,sci1,mid}/*/'
 # and have some 'sound files/a.mp3'
 from tools.sci.sounder.sounder import read_snd_file
 
@@ -21,14 +23,16 @@ def test_read_snd_file_auto_detect(self):
                 for p in game.glob('*'):
                     if p not in ignored_list:
                         with self.assertNoLogs('sounder', level='ERROR'):
-                            result = read_snd_file(p, 'AUTO_DETECT', info=False)
+                            result = read_input(p, 'AUTO_DETECT', input_digital=None, info=False)
                             # print(p, result['input_version'], result['wave'] is not None)
                             if kind.stem == 'sci0_early':
                                 self.assertEqual(result['input_version'], 'SCI0_EARLY')
                             elif kind.stem == 'sci0':
                                 self.assertEqual(result['input_version'], 'SCI0')
                             elif kind.stem == 'sci1':
                                 self.assertEqual(result['input_version'], 'SCI1+')
+                            elif kind.stem == 'mid':
+                                self.assertEqual(type(result['midifile']), mido.MidiFile)
                             else:
                                 self.fail()
 
@@ -39,7 +43,7 @@ def test_save_sci0_early(self):
                 for p in game.glob('*'):
                     if p not in ignored_list:
                         with self.assertNoLogs('sounder', level='ERROR'):
-                            midi_wave = read_snd_file(p, 'AUTO_DETECT', info=False)
+                            midi_wave = read_input(p, 'AUTO_DETECT', input_digital=None, info=False)
                             save_sci0(midi_wave, p, save_file, is_early=True)
                             read_snd_file(Path(save_file), 'AUTO_DETECT', info=False)
         try:
@@ -54,7 +58,7 @@ def test_save_sci0(self):
                 for p in game.glob('*'):
                     if p not in ignored_list:
                         with self.assertNoLogs('sounder', level='ERROR'):
-                            midi_wave = read_snd_file(p, 'AUTO_DETECT', info=False)
+                            midi_wave = read_input(p, 'AUTO_DETECT', input_digital=None, info=False)
                             save_sci0(midi_wave, p, save_file, is_early=False)
                             read_snd_file(Path(save_file), 'AUTO_DETECT', info=False)
         try:
@@ -84,7 +88,7 @@ def test_save_midi(self):
                 for p in game.glob('*'):
                     if p not in ignored_list:
                         with self.assertNoLogs('sounder', level='ERROR'):
-                            midi_wave = read_snd_file(p, 'AUTO_DETECT', info=False)
+                            midi_wave = read_input(p, 'AUTO_DETECT', input_digital=None, info=False)
                             save_midi(midi_wave, p, save_file, save_midi_device='ALL CHANNELS IN FILE')
                             read_midi_file(Path(save_file))
         try:
@@ -99,7 +103,7 @@ def test_save_wav(self):
                 for p in game.glob('*'):
                     if p not in ignored_list:
                         with self.assertNoLogs('sounder', level='ERROR'):
-                            midi_wave = read_snd_file(p, 'AUTO_DETECT', info=False)
+                            midi_wave = read_input(p, 'AUTO_DETECT', input_digital=None, info=False)
                             save_wave(midi_wave['wave'], p, save_file)
         try:
             Path(save_file).unlink()