Laborejo/engine/items.py

# -*- coding: utf-8 -*-
"""
Copyright 2017, Nils Hilbricht, Germany ( https://www.hilbricht.net )

This file is part of Laborejo ( https://www.laborejo.org )

Laborejo is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
import logging; logging.info("import {}".format(__file__))

#Standard Library Modules
from collections import deque
from fractions import Fraction
from weakref import WeakSet, WeakValueDictionary
from warnings import warn

#Third Party Modules
from calfbox import cbox

#Template Modules
import template.engine.pitch as pitchmath
import template.engine.duration as duration
from template.engine.duration import DM, DB, DL, D1, D2, D4, D8, D16, D32, D64, D128, D256, D512, D1024, D_DEFAULT, D_STACCATO, D_TENUTO, D_TIE
from template.helper import EndlessGenerator, flatList

##########################
## Part of other items ##
##########################

class Note(object):
    """A note cannot be inserted in a track directly. It relies on a Chord structure.

    A note is not the level of operation. All note operations go through the notes parent Chord
    item. This is not only a design but is actually needed for undo. Only the Chord methods return
    their reverse operation for the undo system to register.

    for example note.octaveUp just sets self.pitch an octave up and returns None."""

    allNotes = WeakValueDictionary() #key is the noteId, value is the weak reference to the Note. AND NOT CHORDS

    def __init__(self, parentChord, duration, pitch):
        self.pitch = pitch #a value from constants.py with octave modifier
        self.duration = duration  #a Duration instance
        self.dynamic = Dynamic()
        self._secondInit(parentChord)

    def _secondInit(self, parentChord):
        """see Item._secondInit"""
        self.parentChord = parentChord
        self.rememberNote()

    @property
    def pitch(self):
        if self._pitch < 20:
            warn("Pitch {} lower than the lowest note detected. Exported normalized non-destructively to lowest possible note. Please raise pitch again manually before further editing.".format(self._pitch))
            return 20
        elif self._pitch > 3140:
            warn("Pitch {} higher than the highest note detected. Exported normalized non-destructively to highest possible note. Please raise pitch again manually before further editing.".format(self._pitch))
            return 3140
        else:
            return self._pitch

    @pitch.setter
    def pitch(self, value):
        self._pitch = value

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData"""
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self._pitch = int(serializedObject["pitch"])
        self.duration = Duration.instanceFromSerializedData(serializedObject["duration"], parentObject = self)
        self.dynamic = Dynamic.instanceFromSerializedData(serializedObject["dynamic"], parentObject = self)
        self._secondInit(parentChord = parentObject)
        return self

    def serialize(self):
        result = {}
        result["class"] = "Note"
        result["duration"] = self.duration.serialize()
        result["pitch"] = self._pitch
        result["dynamic"] = self.dynamic.serialize()
        return result

    def rememberNote(self):
        oid = id(self)
        Note.allNotes[oid] = self
        #weakref_finalize(self, print, "deleted note "+str(oid))
        return oid

    def copy(self, newParentChord):
        new = Note(newParentChord, self.duration.copy(), self._pitch)
        new.dynamic = self.dynamic.copy()
        return new

    def __lt__(self, other): # self < other
        return self.pitch < other.pitch
    def __le__(self, other): # self <= other
        return self.pitch <= other.pitch
    def __eq__(self, other): # self == other
        return self.pitch == other.pitch
    def __ne__(self, other): # self != other
        return self.pitch != other.pitch
    def __gt__(self, other): # self > other
        return self.pitch > other.pitch
    def __ge__(self, other): # self >= other
        return self.pitch >= other.pitch

    def asDotOnLine(self, clef):
        return pitchmath.distanceInDiatonicStepsFrom1720[self.pitch] + clef.asDotOnLineOffset

    def accidental(self, keysig):
        """-20, -10, 0, +10, +20
        and just 1, which is Natural"""
        diff = pitchmath.diffToKey(self.pitch, keysig)

        #If there is a variation to the keysig. We need to find out if a white note has to get an accidental or an already altered noted (from the keysig) gets a natural sign.
        if diff and self.pitch == pitchmath.toWhite[self.pitch]:
            return 1 #natural
        elif diff == 0: #This just means it is the same as the keysig.
            return diff
        else: #it is different to the keysig. Traditionally the accidental shows the difference to white, not to the keysig. eis in C-minor is #e not Xe/##e
            return self.pitch - pitchmath.toWhite[self.pitch]

    #Inplace pitch modifications
    def sharpen(self):
        self.pitch = pitchmath.sharpen(self.pitch)

    def flatten(self):
        self.pitch = pitchmath.flatten(self.pitch)

    def intervalAutomatic(self, rootPitch, targetPitch):
        """Change the notepitch with the same interval that is between
        rootPitch and targetPitch"""
        self.pitch = pitchmath.intervalAutomatic(self.pitch, rootPitch, targetPitch)

    def mirrorAroundCursor(self, cursorPitch):
        self.pitch = pitchmath.mirror(self.pitch, axis=cursorPitch)

    #def toWhite(self):
    #    self.pitch = pitch.toWhite[self.pitch]

    #def mirror(self, keysig, axis):
    #    """Wants two pitches"""
    #    self.pitch = pitch.mirror(self.pitch, axis)
    #    self.toScale(keysig)

    def octaveUp(self):
        self.pitch = self.pitch + 350

    def octaveDown(self):
        self.pitch = self.pitch - 350

    def toScale(self, keysig):
        self.pitch = pitchmath.toScale(self.pitch, keysig)

    def stepUpInScale(self, keysig):
        self.pitch += 50
        self.toScale(keysig) #adjust to keysig.

    def stepDownInScale(self, keysig):
        self.pitch -= 50
        self.toScale(keysig) #adjust to keysig.

    def exportObject(self, trackState):
        dur = self.duration.completeDuration()
        on, off = self.duration.noteOnAndOff(trackState, dur)
        result = {
            "type" : "note",
            "id" : id(self),
            "accidental" : self.accidental(trackState.keySignature()),
            "dotOnLine" : self.asDotOnLine(trackState.clef()),
            "dots" : self.duration.dots,
            "tuplets" : self.duration.tuplets,
            "notehead" : self.duration.notehead,

            "completeDuration" : dur,
            #"leftModInTicks" : self.duration.startModInTicks(trackState), #only for gui. midi is in self.noteOnAndOff
            "leftModInTicks" : on,
            #"rightModInTicks" : self.duration.endModInTicks(trackState), #only for gui. midi is in self.noteOnAndOff
            "rightModInTicks" : off - dur,

            "tieDistanceInTicks" : dur, #because technically that is right. This note sounds until this value.
            "tie" : "", #empty, first, notFirst  . notFirst is the end of a tie-sequence as well as a middle tie:   c ~ [c] ~ c

            "manualOverride" : bool(self.duration.shiftStart or (self.duration.shiftEnd and not self.duration.durationKeyword == D_TIE)),
            "durationKeyword" : self.duration.durationKeyword, #an int. We internally use constants through module constants.D_KEYWORD
            "velocity" : self.dynamic.velocity(trackState), #int
            "velocityManualOverride" : bool(self.dynamic.velocityModification),
            "endTick" : trackState.tickindex, #for ties.
            "dynamicKeyword" : self.dynamic.dynamicKeyword, #int/"enum", defined in constants. One-time Dynamic Signature for sfz, fp and accents.
            }

        #Handle tied notes.
        if self.pitch in trackState.EXPORTtiedNoteExportObjectsWaitingForClosing:
            result["tie"] = "notFirst"
            if not self.duration.durationKeyword == D_TIE: #this was the last tied note, not a middle.
                trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch]["rightModInTicks"] = trackState.tickindex - trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch]["endTick"] + result["rightModInTicks"]
                trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch]["tieDistanceInTicks"] = trackState.tickindex - dur - trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch]["endTick"] + trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch]["completeDuration"]

                del trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch] #delete this pitch from the dict. Obviously this does not delete the exportObject.
        elif self.duration.durationKeyword == D_TIE: #since we already made sure this pitch is not waiting for a closing/middle tie but we are a tie we can be sure to be the first in a sequence.
            result["tie"] = "first"
            trackState.EXPORTtiedNoteExportObjectsWaitingForClosing[self.pitch] = result

        self._cachedExportObject = result

        return result

    def lilypond(self):
        """Called by chord.lilypond(), See Item.lilypond for the general docstring.
        Returns two strings, pitch and duration.
        The duration is per-chord or needs special polyphony."""
        return pitchmath.pitch2ly[self.pitch], self.duration.lilypond()

class Dynamic(object):
    """dynamic means velocity midi terms.
    (but not volume and not expression).

    Absolute velocity can never go over 128 or below 0. 0 means mute.

    This is not a Dynamic Item which can be inserted into a track.

    Only notes have a working dynamic attribute.
    """
    def __init__(self):
        self.velocityModification = 0 #signed int which will simply get added to the current dynamic signatures value.
        self.dynamicKeyword = 0 #TODO One-time Dynamic Signature for sfz, fp and accents.

    def _secondInit(self, parentNote):
        """see Item._secondInit"""
        pass

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData"""
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self.velocityModification = int(serializedObject["velocityModification"])
        self.dynamicKeyword = int(serializedObject["dynamicKeyword"])
        self._secondInit(parentNote = parentObject)
        return self

    def serialize(self):
        result = {}
        result["class"] = "Dynamic"
        result["velocityModification"] = self.velocityModification
        result["dynamicKeyword"] = self.dynamicKeyword
        return result

    def copy(self):
        new = Dynamic()
        new.velocityModification = self.velocityModification #int, needs no special copy
        new.dynamicKeyword = self.dynamicKeyword  #int, needs no special copy
        return new

    def velocity(self, trackState):
        #todo: if keyword. see init
        base = trackState.dynamicSignature().baseVelocity(trackState)
        if trackState.dynamicRamp: #maybe override the baseVelocity.
            vel = trackState.dynamicRamp.getVelocity(trackState.tickindex - trackState.track.previousItem().logicalDuration() ) #TODO: The tickindex is already ahead of the current item. Somehow in the whole program that doesn't matter except for DynamicRamps. this is really strange. I don't know if this is a design error or one of the few exceptions. Actually, a rather prominent "exception" is the exportObject for every item which is state.tickindex - item.logicalDuration() and even has a comment. So I guess it is alright.
            if not vel is None: #0 is good. None means there is a DynamicRamp but not a target velocity. So we ignore it for now.
                base = vel

        value = int(base + self.velocityModification)
        if value < 0:
            return 0
        elif value > 127:
            return 127
        else:
            return value

class Duration(object):
    """Don't be confused by the amount of durations in the score.
    Each item has a duration. A clef has a duration, including the default one in TrackState.
    A chord has a null-duration itself, because it is an Item, and also each note in the chord
    has its own duration. These are the real durations.

    There are multiple concepts of "duration" in the program:

    baseDuration: The note without dots, tuplets or other modifications. This is the basic note
    on a sheet of paper.

    completeDuration: baseDuration with dots and tuplets. But no shifting(staccato, manual mod etc.)

    noteOnAndOff: the final tick value for midi export. complete duration with all
    expression modifications like staccato, slurs or tenuto and user fine control

    logicalDuration: Value used to calculate the order and spacing of items. aka. determines how
    many ticks the cursor advances when parsing the item or how many pixels space are needed in a
    GUI before the next item. For normal chords this is the same as completeDuration.
    But this also includes values for MultiMeasure Rests and other special cases.
    Note that this is not the sounding duration from noteOnAndOff but the "original" duration
    without any interpretation or expression based modifications.

    The duration of a chord is called "durationGroup", which is it's own class. Why the name
    mismatch? Because all the data that matters to the outside gets exported to a static dict.

    For chords the logicalDuration is the shortest logical note. Example: A chord consists of
    a half note with staccato and a quarter note which is extended by quite a bit through manual
    shifting. The sounding quarter note is longer than the sounding half note.
    noteOnAndOff(half note) < noteOnAndOff(quarter).
    But the logical duration is still the completeDuration where a half note is always longer than
    a quarter note.
    """

    def __init__(self, baseDuration):
        self._baseDuration = baseDuration #base value. Without dots, tuplets/times , overrides etc. this is the duration for all representations
        self.tuplets = [] # a list of tuplets [(numerator, denominator), (,)...]. normal triplet is (2,3). Numerator is the level of the notehead. 2^n Each is one level of tuplets, arbitrary nesting depth. [2,3] for triplet # Tuplet multiplication is used for all representations and gets auto-merged on lilypond output to tuplet groups (if its in a chord).   This is a list and not tuple() because json load will make it a list anyway.
        self.dots = 0 #number of dots.
        self.durationKeyword = D_DEFAULT
        #The offsets shift the start and ending to the left and right or rather: earlier and later. positive values mean later, negative values mean earlier.
        self.shiftStart = 0 # in ticks
        self.shiftEnd = 0 # in ticks
        self._secondInit(parentItem = None) #see Item._secondInit

    def _secondInit(self, parentItem):
        """see Score._secondInit"""
        #discard parentItem
        self.genericNumber = self.calcGenericNumber()
        self.notehead = self.calcNotehead() #depends on self.genericNumber
        #self.cachedCompleteDuration = None

    def __setattr__(self, name, value):
        #print ("change!", self, name, value)
        super().__setattr__(name, value)
        super().__setattr__("cachedCompleteDuration", None)


    @classmethod
    def createByGuessing(cls, completeDuration): #class method. no self.
        """Return a new Duration. A constructor class method.
        Take one absolute tick value and guess what is the best
        duration combination (base, tuplets, dots...) for it.

        createByGuessing exists to distinguish between dotted notes and fractions.
        This means that the duration must be somehow musical precise and without fuzzyness.
        You can't shove arbitrary live-recorded duration values in here. It is not a quantizer"""

        def _closest(num, datalist):
            """only for positive numbers"""
            if num in datalist:
                return num
            else:
                positions = len(datalist)
                tmplist = [0] + sorted(datalist)
                tmplist.reverse()
                #Loop over all positions except the last 0.

                for x in range(positions):
                    if tmplist[x+1] < num < tmplist[x]:
                        return tmplist[x]
                else: #the highest value in datalist was smaller
                    return None

        durList = [D1024, D512, D256, D128, D64, D32, D16, D8, D4, D2, D1, DB, DL, DM]
        guessedBase = _closest(completeDuration, durList)

        if guessedBase:
            if completeDuration in durList:  #no need for guessing
                new = cls(completeDuration)
            elif completeDuration/1.5 in durList:  #single dot
                new = cls(completeDuration/1.5)
                new.dots = 1
            elif completeDuration/1.5/1.5 in durList:  #double dots
                new = cls(completeDuration/1.5/1.5)
                new.dots = 2
            else: #tuplet. That means the value is below a standard "notehead" duration. We need to find how much much lower.
                new = cls(guessedBase)
                #ratio = completeDuration / guessedBase #0.666~ for triplet
                newRatio = Fraction(completeDuration, guessedBase).limit_denominator(100000) #protects 6 or 7 decimal positions
                new.tuplets = [(newRatio.numerator, newRatio.denominator)]
        else:
            #the base could not be guessed. In this case we just create a non-standard note.
            warn("non-standard duration generated")
            new = cls(int(completeDuration)) #210, minimum base duration value.

        return new

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData"""
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self._baseDuration = int(serializedObject["baseDuration"])
        self.tuplets = serializedObject["tuplets"] #TODO: make sure the types are correct?
        self.dots = int(serializedObject["dots"])
        self.durationKeyword = int(serializedObject["durationKeyword"])
        self.shiftStart = int(serializedObject["shiftStart"])
        self.shiftEnd = int(serializedObject["shiftEnd"])
        self._secondInit(parentItem = parentObject)
        return self

    def serialize(self):
        result = {}
        result["class"] = "Duration"
        result["baseDuration"] = self._baseDuration
        result["tuplets"] = self.tuplets
        result["dots"] = self.dots
        result["shiftStart"] = self.shiftStart
        result["shiftEnd"] = self.shiftEnd
        result["durationKeyword"] = self.durationKeyword
        return result

    @property
    def baseDuration(self):
        return self._baseDuration

    @baseDuration.setter
    def baseDuration(self, value):
        """implement limits"""
        if value < D1024:
            value = D1024
        elif value > DM:
            value = DM
        self._baseDuration = value

    def copy(self):
        new = Duration(self.baseDuration)
        new.tuplets = self.tuplets.copy()
        new.dots = self.dots
        new.genericNumber = new.calcGenericNumber()
        new.notehead = new.calcNotehead()
        new.durationKeyword = self.durationKeyword
        new.shiftStart = self.shiftStart
        new.shiftEnd = self.shiftEnd
        return new

    def augment(self):
        self.baseDuration *= 2
        self.genericNumber = self.calcGenericNumber()
        self.notehead = self.calcNotehead()

    def diminish(self):
        self.baseDuration /= 2
        self.genericNumber = self.calcGenericNumber()
        self.notehead = self.calcNotehead()

    def calcGenericNumber(self):
        """A number for humans and lilypond. 4 = quarter, 2 = half"""
        return duration.baseDurationToTraditionalNumber[self.baseDuration]

    def calcNotehead(self):
        """What classical notehead would be used
        to express baseDuration"""
        if self.genericNumber >= 4:
            return 4
        else:
            return self.genericNumber

    def completeDuration(self):
        """Ticks with dots and tuplets. But not leftMod rightMod
         x = basic note value , n = number of dots: 2x - x/2^n

        This function doesn't seem to be much. Why all the optimisation?
        Because it is called very very often. many times or so for a single note.
        """
        if not self.baseDuration:
            return 0

        if self.cachedCompleteDuration:
            return self.cachedCompleteDuration
        else:
            value = 2 * self.baseDuration - self.baseDuration / 2**self.dots
            for numerator, denominator in self.tuplets:
                value = value * numerator / denominator

            if not value == int(value):
                raise ValueError("Only integers are allowed as durations. {} is not {}.".format(value, int(value)))  #TODO: leave this in until real world testing. We *could* live with floats for very complex tuplets, but let's see where this leads us.

            value = int(value)
            super().__setattr__("cachedCompleteDuration", value)
            return value

    def noteOnAndOff(self, trackState, completeDuration):
        """For GUI and Midi
        return a tick value, the offset from an imaginary
        starting point on the tick/time axis, which needs to be
        added later. Think t=0 during this function.

        We do not export the actual duration since this function
        is intended for midi which only needs start and end points
        and calculates the duration itself."""
        on = self.startModInTicks(trackState)(completeDuration)
        off = completeDuration + self.endModInTicks(trackState)(completeDuration)
        if on > off:
            warn("Your calculation resulted in a note off before note on. Please change it manually. Forced to standard note on/off.\n{}\n{}".format(self.startModInTicks(trackState), self.endModInTicks(trackState)))
            return 0, completeDuration
        else:
            return (on, off)

    def startModInTicks(self, trackState):
        """Modifications have different priority. A user created mod
        is always highest priority.
        Contexts like slurs or other instructions are next.
        Then follow individual duration keywords. There can only be one
        keyword at a time."""
        if self.shiftStart:
            return lambda x: self.shiftStart
        elif trackState.duringLegatoSlur:
            return trackState.track.durationSettingsSignature.legatoOn
        elif self.durationKeyword == D_DEFAULT or self.durationKeyword ==  D_TIE:
            return trackState.track.durationSettingsSignature.defaultOn
        elif self.durationKeyword ==  D_STACCATO:
            return trackState.track.durationSettingsSignature.staccatoOn
        elif self.durationKeyword == D_TENUTO:
            return trackState.track.durationSettingsSignature.tenutoOn
        else:
            raise ValueError("Duration Keyword {} unknown".format(self.durationKeyword))

    def endModInTicks(self, trackState):
        """see Duration.startMod"""
        if self.shiftEnd:
            return lambda x: self.shiftEnd
        elif trackState.duringLegatoSlur:
            if self.durationKeyword == D_STACCATO: #slur plus staccato cancels each other out
                return trackState.track.durationSettingsSignature.defaultOff
            else:
                return trackState.track.durationSettingsSignature.legatoOff
        elif self.durationKeyword == D_DEFAULT or self.durationKeyword == D_TIE:
            return trackState.track.durationSettingsSignature.defaultOff
        elif self.durationKeyword ==  D_STACCATO:
            return trackState.track.durationSettingsSignature.staccatoOff
        elif self.durationKeyword == D_TENUTO:
            return trackState.track.durationSettingsSignature.tenutoOff
        else:
            raise ValueError("Duration Keyword {} unknown".format(self.durationKeyword))

    def toggleDurationKeyword(self, keywordConstant):
        """Activate a keyword for a note. If the note already has one,
        overwrite it. Except it already has the same keyword, then
        remove it"""
        if self.durationKeyword == keywordConstant:
            self.durationKeyword = D_DEFAULT
        else:
            self.durationKeyword = keywordConstant

    def toggleDot(self):
        if self.dots == 1:
            self.dots = 2
        elif self.dots == 0:
            self.dots = 1
        else:
            self.dots = 0

    def toggleTriplet(self):
        if self.tuplets == [(2,3)]:
            self.tuplets = []
        else:
            self.tuplets = [(2,3)]

    def lilypond(self):
        """Called by note.lilypond(), See Item.lilypond for the general docstring.
        returns a number as string."""
        
        if self.durationKeyword == D_TIE:
            append = "~"
        else:
            append = ""
        
        n = self.genericNumber
        if n == 0:
            return "\\breve" + append
        elif n == -1:
            return "\\longa" + append
        elif n == -2:
            return "\\maxima" + append
        else:
            return str(n) + self.dots*"." + append

class DurationGroup(object):
    """Holds several durations and returns values meant for chords.
    Always up to date since notelist is a mutable data type.

    Is compatible to Duration() methods and parameters.

    For the representation and calculation always the minimum
    note duration counts for the whole chord."""
    def __init__(self, chord):
        self.chord = chord
        self.group = True
        self.minimumNote = None
        self.cacheMinimumNote() #the shortest note. set inplace.

    def cacheMinimumNote(self):
        work = []
        for note in self.chord.notelist:
            work.append((note.duration.completeDuration(), note))
        self.minimumNote =  min(work)[1]  #a list of tuplets (duration, note). The minimum is the one with the lowest duration, [1] is the note

    def templateDuration(self):
        """for chord addNote we need some template"""
        return self.minimumNote.duration.copy()

    def completeDuration(self):
        """Complete Musical Duration of this item."""
        return self.minimumNote.duration.completeDuration()

    def hasTuplet(self):
        """Return true if there is any tuplet in any note"""
        return any(note.duration.tuplets for note in self.chord.notelist)

    @property
    def baseDuration(self):
        return self.minimumNote.duration.baseDuration

    @property
    def genericNumber(self):
        return duration.baseDurationToTraditionalNumber[self.baseDuration]


##############################################
## Items ##
##############################################


def createChordOrRest(completeDuration, pitchlist, velocityModification):
        """Return a Chord or Rest instance.
        The complete duration is the absolute duration after
        dots, tuplets etc. e.g. generated by midi in.
        Pitchlist can also be empty then it generates rests. But you
        have to explicitely set it to empty.

        This is used by the pattern generator which supports both
        rests and chords and works by absolute durations.
        """

        durationObject = Duration.createByGuessing(completeDuration)
        dynamic = Dynamic()
        dynamic.velocityModification = velocityModification
        if pitchlist: #chord
            new = Chord(firstDuration=durationObject, firstPitch=pitchlist[0])
            for pitch in pitchlist[1:]:
                new.addNote(pitch)
            for note in new.notelist:
                note.dynamic = dynamic.copy()
        else:
            new = Rest(durationObject)

        return new

class Item(object):

    pseudoDuration = Duration(0) #any item has a duration. But we don't need a new pseudo duration instance for every clef and keysig.

    def __init__(self):
        self.duration = self.pseudoDuration
        self.notelist = []
        self.parentBlocks = WeakSet() #this is filled and manipulated by the Block Class through copy, split, contentLink etc. this takes also care of Item.copy(). Don't look in this file for anything parentBlocks related.
        #self._secondInit(parentBlock = None) #Items don't know their parent object. They are in a mutable list which can be in several containers.
        self.lilypondParameters = { # Only use basic data types! Not all lilypond parameters are used by every item. For clarity and editing  we keep them all in one place.
            "visible" : True,
            "override" : "",
            "explicit" : False,
            }

    def _secondInit(self, parentBlock):
        """see Score._secondInit"""        

    def deserializeDurationAndNotelistInPlace(self, serializedObject):
        """Part of instanceFromSerializedData.
        Any Item has a notelist and a duration, even if they are
        empty or never read.

        Needs a self, so
        the instance needs to be created. Call after
        self = cls.__new__(cls)"""
        self.parentBlocks = WeakSet() #inserted by the parent load method
        self.duration = Duration.instanceFromSerializedData(serializedObject["duration"], parentObject = self) #needs to be implemented in every childclass because we do not call Item.instanceFromSerializedData
        self.notelist = [Note.instanceFromSerializedData(note, parentObject = self) for note in serializedObject["notelist"]]
        self.lilypondParameters = serializedObject["lilypondParameters"]

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData.
        This does not get called by a child class so you need to
        reimplement self.duration and other common parameters in
        every child class!"""
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self.deserializeDurationAndNotelistInPlace(serializedObject) #creates parentBlocks
        self._secondInit(parentTrack = parentObject)
        return self

    def serialize(self):
        """Return a serialized data from this instance. Used for save, load
        and undo/redo.
        Can be called in a chain by subclasses with super().serialize

        The main difference between serialize and exportObject is that
        serialize does not compute anything. I just saves the state
        without calulating note on and off or stem directions,
        for example.
        """
        #result = super().serialize() #call this in child classes
        result = {}
        result["class"] = self.__class__.__name__
        result["duration"] =  self.duration.serialize()
        result["notelist"] = [note.serialize() for note in self.notelist]
        result["lilypondParameters"] = self.lilypondParameters
        return result

    def _exportObject(self, trackState):
        """Implement this in every child class, return a dict.

        Base class function to return a dict which is a good layout
        export basis. For example for GUI frontends. They don't have to
        parse and calculate their own values in slow pure Python then

        midiBytes is a list with patternBlob/binary data generated by
        cbox.Pattern.serialize_event(position, midibyte1 (noteon), midibyte2(pitch), midibyte3(velocity))
        Items with duration, like chords, will most like need to create
        at least two patternBlobs, one for the start (e.g. note on) and one
        for the stop (e.g. note off)
        """
        raise NotImplementedError("Implement _exportObject() for this item type!")

        #And for documentation and education here a template exportObject
        duration = 0
        return {
            "type" : "",
            "completeDuration" : duration,
            "tickindex" : trackState.tickindex - duration, #we parse the tickindex after we stepped over the item.
            "midiBytes" : [],
            "UIstring" : "", #this is for a UI, possibly a text UI, maybe for simple items of a GUI. Make it as short and unambigious as possible.
        }

    def exportObject(self, trackState):
        exportDict = {
            "id" : id(self),
            "lilypondParameters" : self.lilypondParameters,
        }

        exportDict.update(self._exportObject(trackState))
        return exportDict


    @property
    def parentTracks(self):        
        #return (block.parentTrack for block in self.parentBlocks[0])
        return (block.parentTrack for block in list(self.parentBlocks)[0].linkedContentBlocksInScore())

    @property
    def parentTrackIds(self):
        return (id(tr) for tr in self.parentTracks)

    def _copy(self):
        """return an independent copy of self"""
        raise NotImplementedError("Implement _copy() for this item type!")

    def copy(self):
        new = self._copy()
        new.lilypondParameters = self.lilypondParameters.copy()
        return new

    def copyParentBlocks(self, oldItem):
        """Move the oldItems parentBlocks to the new item"""        
        #self.parentBlocks = oldItem.parentBlocks #TODO. We took that out when pasting after deleting a track and recreating failed. Why do we to copy the parentBlocks when a parentBlock is added during block.insert anyway? Wild guess: we don't. 

    def logicalDuration(self):
        return 0

    def deserializeInplace(self, serializedObject):
        """Change this instance values to the values in the
         serialized object. Part of the undo/redo system"""
        return

    #Empty methods that allow for apply to selection and other iterators to work with items that cannot respond to a command. e.g. stepUp on a rest.
    #Api functions must react to these function returning None. For example Undo will not register anything when no undo-function is returned by an item-method.
    #We don't use functions(self, *args) and then a bunch of function1 = function2 = function3 because this way it is easier to see what methods are available (and what arguments they need) when creating new classes from Item.

    def sharpenNoteNearPitch(self, pitch): pass
    def flattenNoteNearPitch(self, pitch): pass
    def stepUpNoteNearPitch(self, pitch, keysig): pass
    def stepDownNoteNearPitch(self, pitch, keysig): pass
    def stepUpOctaveNoteNearPitch(self, pitch): pass
    def stepDownOctaveNoteNearPitch(self, pitch): pass
    def augmentNoteNearPitch(self, pitch): pass
    def diminishNoteNearPitch(self, pitch): pass
    def toggleDotNearPitch(self, pitch): pass
    def toggleTripletNearPitch(self, pitch): pass
    def setTupletNearPitch(self, pitch, tupletListForDuration): pass
    def toggleDurationKeywordNearPitch(self, pitch, keywordConstant): pass
    def moreVelocityNearPitch(self, pitch): pass
    def lessVelocityNearPitch(self, pitch): pass
    def moreDurationNearPitch(self, pitch): pass
    def lessDurationNearPitch(self, pitch): pass
    def resetVelocityAndDurationModsNearPitch(self, pitch): pass

    def sharpen(self): pass
    def flatten(self): pass
    def augment(self): pass
    def diminish(self): pass
    def toggleDot(self): pass
    def toggleTriplet(self): pass
    def setTuplet(self, durationTupletListForEachNote): pass
    def toggleDurationKeyword(self, listOfKeywordConstants): pass
    def stepUp(self, keysigList): pass
    def stepDown(self, keysigList): pass
    def stepUpOctave(self): pass
    def stepDownOctave(self): pass
    def intervalAutomatic(self, rootPitch, targetPitch): pass
    def split(self, newparts): pass
    def addNote(self, pitch): pass
    def deleteNote(self, note): pass
    def removeNoteNearPitch(self, pitch): pass
    def moreVelocity(self): pass
    def lessVelocity(self): pass
    def moreDuration(self): pass
    def lessDuration(self): pass
    def resetVelocityAndDurationMods(self): pass

    def toggleBeam(self): pass
    def removeBeam(self): pass

    def midiRelativeChannelPlus(self): pass
    def midiRelativeChannelMinus(self): pass
    def midiRelativeChannelReset(self): pass

    def mirrorAroundCursor(self, cursorPitch): pass

    def lilypond(self):
        if self.lilypondParameters["override"]:
            return self.lilypondParameters["override"]
        else:
            return self._lilypond()

    def _lilypond(self):
        """called by block.lilypond(), returns a string.
        Don't create white-spaces yourself, this is done by the structures.
        When in doubt prefer functionality and robustness over 'beautiful' lilypond syntax."""
        return ""

class TemplateItem(Item):
    """Use this with copy and paste to create a new item class"""

    def __init__(self, itemData):
        """Either init gets called, by creation during runtime, or instanceFromSerializedData.
        That means everything in init must be matched by a loading call in instanceFromSerializedData."""
        super().__init__()
        raise NotImplementedError("Implement __init__() for this item type!")
        self.itemData = itemData #matches instanceFromSerializedData and serialize
        self._secondInit(parentBlock = None) #see Item._secondInit.

    def _secondInit(self, parentBlock):
        """After creating a new item during runtime or after loading from a file this gets called.
        Put every calcuation that depends on values that could be changed by the user.
        see Item._secondInit"""
        super()._secondInit(parentBlock) #Item._secondInit

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData"""
        raise NotImplementedError("Implement instanceFromSerializedData() for this item type!")
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self.itemData = serializedObject["itemData"]
        self.deserializeDurationAndNotelistInPlace(serializedObject)
        self._secondInit(parentBlock = parentObject)
        return self

    def serialize(self):
        raise NotImplementedError("Implement serialize() for this item type!")
        result = super().serialize() #call this in child classes
        result["itemData"] = self.itemData
        return result

    def _copy(self):
        """return an independent copy of self"""
        raise NotImplementedError("Implement _copy() for this item type!")
        new = TemplateItem(self.itemData)
        return new


    def _exportObject(self, trackState):
        """Implement this in every child class, return a dict.

        Base class function to return a dict which is a good layout
        export basis. For example for GUI frontends. They don't have to
        parse and calculate their own values in slow pure Python then

        midiBytes is a list with patternBlob/binary data generated by
        cbox.Pattern.serialize_event(position, midibyte1 (noteon), midibyte2(pitch), midibyte3(velocity))
        Items with duration, like chords, will most like need to create
        at least two patternBlobs, one for the start (e.g. note on) and one
        for the stop (e.g. note off)
        """
        raise NotImplementedError("Implement _exportObject() for this item type!")

        #And for documentation and education here a template exportObject
        duration = 0
        return {
            "type" : "",
            "completeDuration" : duration,
            "tickindex" : trackState.tickindex - duration, #we parse the tickindex after we stepped over the item.
            "midiBytes" : [],
            "UIstring" : "", #this is for a UI, possibly a text UI, maybe for simple items of a GUI. Make it as short and unambigious as possible.
        }

    def _lilypond(self):
        """called by block.lilypond(), returns a string.
        Don't create white-spaces yourself, this is done by the structures.
        When in doubt prefer functionality and robustness over 'beautiful' lilypond syntax."""
        return ""

class Chord(Item):
    """Duration and firstPitch are for the first note.
    You must only modify the notelist through Chord methods. The chord methods keep
    the list sorted and also keep track what is the shortest and longest note, and other meta-data.
    Replacing .notelist or using list methods like .append directly will break the whole program.
    Especially the lowest and shortest notes are used multiple times on every item export!
    """

    def __init__(self, firstDuration, firstPitch):
        super().__init__()
        firstNote = Note(self, firstDuration, firstPitch)
        self.notelist = [firstNote]
        self._beamGroup = False # Works a bit like legato slurs, only this is a property of a note, not an item.   self.beamGroup is a @property that self-corrects against errors.
        self._secondInit(parentBlock = None) #see Item._secondInit.
        self.midiChannelOffset = 0 #from -15 to 15. In reality much less. Expected values are -3 to +3

    def _secondInit(self, parentBlock):
        """see Item._secondInit"""
        super()._secondInit(parentBlock) #Item._secondInit
        self.durationGroup = DurationGroup(self) #durationGroup is a dynamic data type and therefore does not need serialize. The serialized durations are in the notelist.
        self._cachedClefForLedgerLines = None

    @classmethod
    def instanceFromSerializedData(cls, serializedObject, parentObject):
        """see Score.instanceFromSerializedData"""
        assert cls.__name__ == serializedObject["class"]
        self = cls.__new__(cls)
        self.deserializeDurationAndNotelistInPlace(serializedObject)
        self._beamGroup = bool(serializedObject["beamGroup"])
        self.midiChannelOffset = serializedObject["midiChannelOffset"]
        self._secondInit(parentBlock = parentObject)
        return self

    def serialize(self):
        result = super().serialize() #call this in child classes
        result["beamGroup"] = self._beamGroup
        result["midiChannelOffset"] = self.midiChannelOffset
        #durationGroup is a dynamic data type and gets its values from its parent chord: self.
        return result

    def logicalDuration(self):
        """Return the logical duration that is used to calculate
        the space and subsequent position of items."""
        #TODO: this gets called pretty often. 6 times per note insert on a previously empty track. Why?!
        return self.durationGroup.completeDuration()

    def _copy(self):
        new = Chord(Duration(0), 0)

        new.copyParentBlocks(self)
        new._beamGroup = self._beamGroup
        new.notelist = []
        for note in self.notelist:
            new.notelist.append(note.copy(new))
        new.durationGroup.cacheMinimumNote()
        return new

    def pitchlist(self):
        return [note.pitch for note in self.notelist] #needs to be subscriptable

    def whitePitchlist(self):
        result = []
        for note in self.notelist:
            result.append(pitchmath.toWhite[note.pitch])
        return result

    def _setNotelist(self, notelist):
        oldNotelist = self.notelist.copy()
        self.notelist = notelist
        self.durationGroup.cacheMinimumNote()
        return lambda: self._setNotelist(oldNotelist)

    def addNote(self, pitch):
        """The notelist gets sorted after insert.
        Each white note is only allowed once in each chord.
        That means gis and ges are not allowed in the same chord.
        And not two times the same note.

        We use the template duration from the group. That means the new
        note has the same main-properties as the existing notes.
        """
        note = Note(self, self.durationGroup.templateDuration(), pitch)
        if pitchmath.toWhite[pitch] in self.whitePitchlist():
            return False #only one of each base note in a chord. no cis, ces and c in the same chord.
        else:
            oldNotelist = self.notelist.copy() #only copies the order, not the items itself. Which is what we need.
            self.notelist.append(note)
            self.notelist.sort()
            self.durationGroup.cacheMinimumNote()
            return lambda: self._setNotelist(oldNotelist)

    def deleteNote(self, note):
        """Delete the exact note, an instance.
        No search for the pitch or crap like "highest first".
        Notelist remains sorted after delete."""
        if len(self.notelist) > 1:
            oldNotelist = self.notelist.copy()
            self.notelist.remove(note)
            self.durationGroup.cacheMinimumNote()
            #No need for sorting.
            return lambda: self._setNotelist(oldNotelist)

    def removeNoteNearPitch(self, pitch):
        note = self.getNearestNote(pitch)
        return self.deleteNote(note)

    def getNearestNote(self, pitch):
        """Return the note which is nearest.
        The behaviour if two notes of the same pitch are in this chord
        is undefined. Whatever cython/python finds first will get
        returned. It really shouldn't matter."""
        listOfTuples = []
        for note in self.notelist:
            listOfTuples.append((abs(pitch - note.pitch), note))
        note = min(listOfTuples)[1] #[1] of the tuple (pitch,noteinstance)
        return note

    #def Note getHighestNote(self):
    #def Note getLowestNotes(self):


    #Direct functions. Do not need a context, don't need a state.
    #Meant for undo.

    def _setNotePitch(self, note, targetPitch):
        """A direct setter for an exact note,
        not just near a pitch.
        Mostly for undo"""
        oldValueForUndo = note.pitch
        note.pitch = targetPitch
        self._cachedClefForLedgerLines = None
        return lambda: self._setNotePitch(note, oldValueForUndo)

    def _setNoteDuration(self, note, durationInstance):
        """see setNotePitch."""
        assert not note.duration is durationInstance
        oldValueForUndo = note.duration.copy()
        note.duration = durationInstance
        self.durationGroup.cacheMinimumNote()
        return lambda: self._setNoteDuration(note, oldValueForUndo)

    def _setNoteDynamic(self, note, dynamicInstance):
        """see _setNoteDuration."""
        assert not note.dynamic is dynamicInstance
        oldValueForUndo = note.dynamic.copy()
        note.dynamic = dynamicInstance
        return lambda: self._setNoteDynamic(note, oldValueForUndo)

    def _setNoteDynamicAndDuration(self, note, durationInstance, dynamicInstance):
        assert not note.duration is durationInstance
        oldDurationValueForUndo = note.duration.copy()
        note.duration = durationInstance
        self._cachedClefForLedgerLines = None

        assert not note.dynamic is dynamicInstance
        oldDynamicValueForUndo = note.dynamic.copy()
        note.dynamic = dynamicInstance

        return lambda: self._setNoteDynamicAndDuration(note, oldDurationValueForUndo, oldDynamicValueForUndo)


    #Nearest Note:
    #Pitch modifications
    def sharpenNoteNearPitch(self, pitch):
        note = self.getNearestNote(pitch)
        oldValueForUndo = note.pitch
        self.notelist.sort()
        note.sharpen()
        return lambda: self._setNotePitch(note, oldValueForUndo)

    def flattenNoteNearPitch(self, pitch):
        note = self.getNearestNote(pitch)
        oldValueForUndo = note.pitch
        self.notelist.sort()
        note.flatten()
        return lambda: self._setNotePitch(note, oldValueForUndo)

    def stepUpNoteNearPitch(self, pitch, keysig):
        note = self.getNearestNote(pitch)
        oldValueForUndo = note.pitch
        note.stepUpInScale(keysig)