Add temporary fix for sonate sheet music

helpers/measure_helpers.py

@@ -9,13 +9,15 @@
 
 
 def split_measures(barlines: List['Barline'], staff: 'Staff') -> List['Measure']:  # barlines sorted on x
-    x1, x2 = (0, 0)
+    x1, x2 = (staff.x-2, 0)
     measures = []
     for i in range(0, len(barlines) + 1):
         if i == len(barlines):
             x2 = staff.lines[0][2]
         else:
             x2 = barlines[i].x
+            if x2 < x1:
+                continue
         measures.append(Measure(staff, i, x1, x2))
         x1 = x2
     return measures

helpers/staff_fixers.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jul 10 15:29:26 2020
+
+@author: super
+"""
+
+from itertools import groupby
+
+def calc_freq(times):
+    times2 = [t for t in times if t is not None]
+    time_f = [len(list(group)) for key, group in groupby(sorted(times2))]
+    nr_instruments = max(time_f)
+    freq_array = time_f*nr_instruments
+    freq_array2 = []
+    offset = 0
+    for i,t in enumerate(times):
+        if t is None:
+            freq_array2.append(0)
+            offset += 1
+        else:
+            freq_array2.append(freq_array[i-offset])
+    return freq_array2
+
+def fix_staff_relations(staffs):
+    times = [s.nr_timewise for s in staffs]
+    instruments = [s.nr_instrument for s in staffs if s.nr_instrument is not None]
+    nr_instruments = max(instruments)
+
+    freq_array2 = calc_freq(times)
+
+    added_times = []
+    for i in range(len(freq_array2)):
+        if freq_array2[i] == 0:
+            if freq_array2[i-1] == nr_instruments:
+                times[i] = times[i-nr_instruments]+1
+                added_times.append(times[i])
+            else:
+                times[i] = times[i-1]
+            freq_array2 = calc_freq(times)
+        else:
+            times[i] = times[i] + len(list(set(added_times)))
+
+    instrs = []
+    ini = 0
+    inst = 0
+    for i in range(len(times)):
+        if times[i] == ini:
+            inst += 1
+        else:
+            inst = 1
+            ini = times[i]
+        instrs.append(inst)
+        staffs[i].nr_timewise = times[i]
+        staffs[i].nr_instrument = inst
+
+    return instrs
+

helpers/staff_helpers.py

@@ -162,7 +162,10 @@ def detect_staff_lines(img_bar, staff_height):
             if i == len(long_lines) - 1:
                 unique_lines.append(l)
         elif i == len(long_lines) - 1:
-            unique_lines.append(long_lines[i - 1])
+            if skip == False:
+                unique_lines.append(long_lines[i - 1])
+            elif len(unique_lines) < 4:
+                unique_lines.append(l)
         else:
             l1 = long_lines[i - 2]
             l2 = long_lines[i - 1]

main.py

@@ -2,6 +2,7 @@
 from typing import List, Dict
 
 import cv2 as cv
+import imutils
 
 from denoise.denoise import denoise
 from dewarp.dewarp import dewarp
@@ -18,22 +19,28 @@
 from staffs.seperate_staffs import separate_staffs
 from template_matching.template_matching import template_matching, AvailableTemplates, template_matching_array
 from utils.util import imshow
+from helpers.staff_fixers import fix_staff_relations
 
 
 def main():
-    input_folder = 'images/sheets/sonate/'
-    original_img = cv.imread(input_folder+'input.png', cv.IMREAD_COLOR)
-
-    # denoise
-    denoised_image = denoise(original_img, isRgb=True)
-
-    # dewarp
-    dewarped_image = dewarp(denoised_image, isRgb=True)
-    # dewarped_image = cv.imread(input_file, cv.IMREAD_COLOR)  # temporary
-#    cv.imwrite(input_folder+'dewarped.png',dewarped_image) 
+    input_folder = 'images/sheets/fmttm/'
+    preprocess = False
+
+    if preprocess == True:
+        original_img = cv.imread(input_folder+'input.png', cv.IMREAD_COLOR)
+
+        # denoise
+        denoised_image = denoise(original_img, isRgb=True)
 
-    imshow("src", dewarped_image)
-#    dewarped_image = cv.imread(input_folder+'dewarped.png', cv.IMREAD_COLOR)
+        # dewarp
+        dewarped_image = dewarp(denoised_image, isRgb=True)
+
+        # save image
+        cv.imwrite(input_folder+'dewarped.png',dewarped_image) 
+
+    else:
+        # load image
+        dewarped_image = cv.imread(input_folder+'dewarped.png', cv.IMREAD_COLOR)
 
     # separate full sheet music into an image for each staff
     staffs = [Staff(s) for s in separate_staffs(dewarped_image)]
@@ -43,11 +50,34 @@ def main():
     # set threshold for template matching
     all_measures: List[Measure] = []
     all_signatures: Dict[int, 'Time'] = {}
+    all_clefs: Dict[int, 'Clef'] = {}
+
+    # groepeer maten naar parts
+    parts = [s.nr_instrument for s in staffs]
+    if None in parts:
+        parts = fix_staff_relations(staffs)
+    parts = list(set(parts))
+    parts.sort()
 
     for staff_index in range(len(staffs)):
+        print(f"staff {staff_index}")
         current_staff: Staff = staffs[staff_index]
+
+#        #for testing size of template
+#        if staff_index == 0:
+##            print(current_staff.dist)
+#            fclef = cv.imread('images/templates/clefs/hacky-g.png', cv.IMREAD_COLOR)
+#            fclef = imutils.resize(fclef, height=int(current_staff.dist * 4))
+#            fh, fw = fclef.shape[:2]
+#            
+#            imf = current_staff.image.copy()
+#            ystart = current_staff.lines[4][1]
+#            imf[ystart:ystart+fh, 275:275+fw] = fclef
+##            imf[ystart:ystart+4*current_staff.dist, 200:300] = (0,0,255)
+#            imshow('test', imf)
+
         # Generate Time signature objects
-        detected_times = template_matching_array(AvailableTemplates.AllTimes.value, current_staff, 0.7)
+        detected_times = template_matching_array(AvailableTemplates.AllTimes.value, current_staff, 0.5)
         time_objects: List['Time'] = []
         for template in detected_times.keys():
             for match in detected_times[template]:
@@ -65,18 +95,12 @@ def main():
         # Store the last used time signature per voice number for potential later use
         all_signatures[current_staff.nr_instrument] = time_objects[-1]
 
-        # find measures
-        # measure_matches = template_matching_array(AvailableTemplates.AllBarlines.value, current_staff, 0.8)
-        # measure_locations = []
-        # imcopy = current_staff.image.copy()
-        # for template in measure_matches.keys():
-        # for meas in measure_matches[template]:
-        #       measure_locations.append(meas)
-        #       cv.rectangle(imcopy, (meas[0],meas[1]), (meas[0]+template.w, meas[1]+template.h), (0,0,255),2)
-        # cv.imshow('bar lines %d'%staff_index, imcopy)
+#        measure_locations = template_matching(AvailableTemplates.Barline.value, current_staff, 0.8)
+#        barlines = select_barlines(measure_locations, current_staff, AvailableTemplates.Barline.value)
 
-        measure_locations = template_matching(AvailableTemplates.Barline.value, current_staff, 0.8)
-        barlines = select_barlines(measure_locations, current_staff, AvailableTemplates.Barline.value)
+        # find note stems and barlines
+        stem_objects, barlines = find_stems(current_staff)
+
 
         # now first finding noteheads to weed out some incorrect barline matches
         # do template matching for notes and rests (to do: change to groups)
@@ -150,7 +174,7 @@ def main():
 
             if remove == 0:
                 real_clefs.append(c1)
-
+            
         # Associate accidentals with a certain note
         global_key_per_measure: List[Accidental] = []
         for measure in measures:
@@ -168,13 +192,36 @@ def main():
                         key_per_measure.append(accidental)
 
             measure.set_key(Key(key_per_measure))
-
-            relevant_clef = max([clef for clef in real_clefs if clef.x < measure.end], key=lambda clef: clef.x)
+
+            prev_clefs = [clef for clef in real_clefs if clef.x < measure.end]
+
+            if len(prev_clefs) == 0:
+                if current_staff.nr_timewise == 1:
+                    # kan gebruikt worden voor het testen van volgende stappen
+#                    relevant_clef = Clef(measures[0].start, current_staff.lines[4][1], AvailableTemplates.ClefG.value)
+                    raise ValueError('OH BOY NO CLEF WAS DETECTED AT THE START OF THE FIRST LINE SEND HELP')
+                else:
+                    last_clef: 'Clef' = all_clefs[current_staff.nr_instrument]
+                    relevant_clef = last_clef
+            else:
+                relevant_clef = max(prev_clefs, key=lambda clef: clef.x)
+            all_clefs[current_staff.nr_instrument] = relevant_clef
             measure.set_clef(relevant_clef)
             if relevant_clef.x > measure.start:
                 measure.show_clef = True
 
-            relevant_time = max([time for time in time_objects if time.x < measure.end], key=lambda time: time.x)
+            prev_times = [time for time in time_objects if time.x < measure.end]
+            if len(prev_times) == 0:
+                if current_staff.nr_timewise == 1:
+                    # kan gebruikt worden voor het testen van volgende stappen
+#                    relevant_time = Time(measures[0].start, current_staff.lines[4][1], AvailableTemplates.TimeC.value)
+                    raise ValueError('OH BOY NO TIME SIGNATURE WAS DETECTED AT THE START OF THE FIRST LINE SEND HELP')
+                else:
+                    last_time: 'Time' = all_signatures[current_staff.nr_instrument]
+                    relevant_time = last_time
+            else:
+                relevant_time = max(prev_times, key=lambda time: time.x)
+            all_signatures[current_staff.nr_instrument] = relevant_time
             measure.set_time(relevant_time)
 
         time_meas = find_measure(measures, time_objects[0].x)
@@ -208,8 +255,7 @@ def main():
             head_obj.set_note(relevant_measure)
             head_obj.set_key(find_measure(measures, head_obj.x).key)
 
-        # find note stems
-        stem_objects = find_stems(current_staff)
+
         # find note beams
         beam_objects = find_beams(current_staff)
 
@@ -250,12 +296,7 @@ def main():
 
         all_measures += measures
 
-    # groepeer maten naar parts
-    parts = []
-    for s in staffs:
-        parts.append(s.nr_instrument)
-    parts = list(set(parts))
-    parts.sort()
+
 
     meas_per_part = []
     for i in parts:
@@ -298,7 +339,7 @@ def main():
                     add_rest(meas1, obj, voice)
 
     tree = ET.ElementTree(root)
-    with open('input_folder/digitalized.xml', 'wb') as f:
+    with open(input_folder+'digitalized.xml', 'wb') as f:
         f.write(
             '<?xml version="1.0" encoding="UTF-8" standalone="no"?><!DOCTYPE score-partwise PUBLIC "-//Recordare//DTD '
             'MusicXML 3.1Partwise//EN" "http://www.musicxml.org/dtds/partwise.dtd">'.encode(

models/note_objects.py

@@ -44,7 +44,7 @@ def __init__(self, x: int, y: int, template: 'Template', is_local: bool = True):
 
     def find_note(self, measure: 'Measure'):
         pitch = find_pitch(measure.staff, self.x, self.adjusted_y())
-        self.note = measure.note_labels[pitch % 7]
+        self.note = measure.note_labels[pitch % 7] if pitch is not None else 'Unknown'
 
     def set_is_local(self, is_local: bool):
         self.is_local = is_local

models/staff_objects.py

@@ -83,7 +83,7 @@ def find_key(self, group):
 class Time:
     timedict = {'2/2 time':(2, 2), '2/4 time':(2, 4), '3/4 time': (3, 4), '3/8 time': (3, 8), '4/4 time': (4, 4), '5/4 time': (5, 4), '5/8 time': (5, 8),
                 '6/4 time': (6, 4), '6/8 time': (6, 8), '7/8 time': (7, 8), '9/8 time': (9, 8), '12/8 time': (12, 8),
-                '4/4 time C': (4, 4)}
+                '4/4 time C': (4, 4), 'alla breve': (2, 2)}
 
     def __init__(self, x: int, y: int, template: 'Template'):
         self.beats, self.beat_type = self.timedict[template.name]

notes/build_notes_objects.py

@@ -13,6 +13,9 @@
 from helpers.note_helpers import find_pitch
 from models.note_objects import Accidental, Stem, AccidentalTypes, Note
 from template_matching.template_matching import template_matching_array, AvailableTemplates
+from utils.util import imshow
+from helpers.staff_helpers import calc_y
+from models.staff_objects import Barline
 
 if TYPE_CHECKING:
     from models.measure import Measure
@@ -47,13 +50,26 @@ def find_stems(staff: 'Staff') -> List['Stem']:
     #    cv2.imshow('stems', imcopy)
 
     stem_list: List['Stem'] = []
+    line_list = []
     for line in lines2_ver:
         stem_line = line[0]
         stem_list.append(Stem(stem_line[0], stem_line[1], stem_line[2], stem_line[3]))
+        line_list.append(stem_line)
+
+    barlines = select_barlines(line_list, staff)
 
-    return stem_list
+    return stem_list, barlines
 
 
+def select_barlines(lines: (int, int, int, int), staff: 'Staff') -> List['Barline']:
+    barlines = []
+    measure_locations = sorted(lines, key=lambda x: x[0])
+    for meas in measure_locations:
+        if min(meas[1], meas[3]) <= calc_y(staff.lines[4], meas[0])+1 and max(meas[1], meas[3]) >= calc_y(staff.lines[8], meas[0])-1:
+            barlines.append(Barline(meas[0], min(meas[1], meas[3]), max(meas[1], meas[3])))
+
+    return barlines
+
 def detect_accidentals(staff: 'Staff', threshold: float) -> List['Accidental']:
     found_accidentals = template_matching_array(AvailableTemplates.AllAccidentals.value, staff, threshold)
     if len(found_accidentals.keys()) == 0:
@@ -237,8 +253,9 @@ def group_accidentals(accidentals: List['Accidental']) -> List[List['Accidental'
             continue
         if result[group_index][0].is_local == accidentals[i].is_local:
             result[group_index].append(accidentals[i])
-            group_index += 1
-            result[group_index] = [accidentals[i]]
+            if group_index < len(result)-1:
+                group_index += 1
+                result[group_index] = [accidentals[i]]
             continue
         else:
             group_index += 1

notes/find_beams.py

@@ -113,6 +113,8 @@ def pythagoras(p1,p2):
 
     beam_tuples = []
     for beam in beams:
+        if len(beam[0])==0 or len(beam[1])==0:
+            continue
         beam_tup = (beam[0][0],beam[0][1],beam[1][0],beam[1][1])
         if beam_tup not in beam_tuples:
             beam_tuples.append(beam_tup)

template_matching/template_matching.py

@@ -51,8 +51,8 @@ def template_matching_array(templates: List['Template'], staff: 'Staff', thresho
 
 
 class AvailableTemplates(Enum):
-    # Bar lines
-    Barline = Template('barline', 'images/templates/barlines/barline.png', 4)
+#    # Bar lines
+#    Barline = Template('barline', 'images/templates/barlines/barline.png', 4)
 
     # Note heads
     NoteheadClosed = Template('closed_notehead', 'images/templates/noteheads/head-filled.png', 1)
@@ -98,8 +98,10 @@ class AvailableTemplates(Enum):
     ClefG_full = Template(ClefTypes.G_CLEF.name, 'images/templates/clefs/g-clef-with-lines.jpg', 7.5)
     ClefF = Template(ClefTypes.F_CLEF.name, 'images/templates/clefs/f-clef-with-lines.jpg', 3)
     ClefC = Template(ClefTypes.C_CLEF.name, 'images/templates/clefs/c-clef-with-lines.jpg', 4)
+#    ClefF_hacky = Template(ClefTypes.F_CLEF.name, 'images/templates/clefs/hacky-f.png', 4)
+#    ClefG_hacky = Template(ClefTypes.G_CLEF.name, 'images/templates/clefs/hacky-g.png', 4)
 
-    AllClefs = [ClefG, ClefG_full, ClefF, ClefC]
+    AllClefs = [ClefG, ClefG_full, ClefF, ClefC]#, ClefF_hacky, ClefG_hacky]
 
     # Accidentals
     Flat = Template(AccidentalTypes.FLAT.acc_type, 'images/templates/accidentals/flat.jpg', 2.4)
@@ -130,6 +132,8 @@ class AvailableTemplates(Enum):
     Time12_8 = Template('12/8 time', 'images/templates/times/12_8.jpg', 4)
     TimeAllaBreve = Template('alla breve', 'images/templates/times/alla_breve.jpg', 4)
     TimeC = Template('4/4 time C', 'images/templates/times/c.jpg', 4)
+#    Time2_top = Template('2 top', 'images/templates/times/2-top.jpg', 2)
+#    Time2_bottom = Template('2 bottom', 'images/templates/times/2-bottom.jpg', 2)
 
     AllTimes = [Time2_2, Time2_4, Time3_4, Time3_8, Time4_4, Time5_4, Time5_8, Time6_4,
-                Time6_8, Time7_8, Time9_8, Time12_8, TimeAllaBreve, TimeC]
+                Time6_8, Time7_8, Time9_8, Time12_8, TimeAllaBreve, TimeC]#, Time2_top, Time2_bottom]