Patroneo/engine/api.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Copyright 2022, Nils Hilbricht, Germany ( https://www.hilbricht.net )

This file is part of the Laborejo Software Suite ( https://www.laborejo.org ),

This application is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import logging; logger = logging.getLogger(__name__); logger.info("import")

#Standard Library Modules
from typing import List, Set, Dict, Tuple

#Third Party Modules
from template.calfbox import cbox

#Template Modules
import template.engine.api #we need direct access to the module to inject data in the provided structures. but we also need the functions directly. next line:
from template.engine.api import *
from template.engine.duration import baseDurationToTraditionalNumber
from template.helper import compress

#Our Modules
from engine.input_apcmini import apcMiniInput


DEFAULT_FACTOR = 1 #for the GUI.

#Swing Lookup Table for functions and callbacks
_percentToSwing_Table = {}
_swingToPercent_Table = {}
for value in range(-100, 100+1):
    #Lookup table.
    if value == 0:
        result = 0
    elif value > 80: #81% - 100% is  0.33 - 0.5
        result = compress(value, 81, 100, 0.33, 0.5)
    elif value > 30: #31% - 80% is  0.15 - 0.33
        result = compress(value, 31, 80, 0.15, 0.32)
    else:
        result = compress(value, 0, 30, 0.01, 0.14)
    r = round(result,8)
    _percentToSwing_Table[value] = r
    _swingToPercent_Table[r] = value  #TODO: this is risky! it only works because we round to digits and percents are integers.



#New callbacks
class ClientCallbacks(Callbacks): #inherits from the templates api callbacks
    def __init__(self):
        super().__init__()

        self.timeSignatureChanged = []
        self.scoreChanged = []
        self.numberOfMeasuresChanged = []
        self.trackStructureChanged = []
        self.trackMetaDataChanged = []
        self.patternChanged = []
        self.stepChanged = []
        self.removeStep = []
        self.exportCacheChanged = []
        self.subdivisionsChanged = []
        self.quarterNotesPerMinuteChanged = []
        self.loopChanged = []
        self.loopMeasureFactorChanged = []
        self.patternLengthMultiplicatorChanged = []
        self.swingChanged = []
        self.swingPercentChanged = []
        self.currentTrackChanged = []

    def _quarterNotesPerMinuteChanged(self):
        """There is one tempo for the entire song in quarter notes per mintue.
        score.isTransportMaster to False means we do not create our own changes
        and leave everything to the default. Negative values are not possible"""
        if session.data.tempoMap.isTransportMaster:
            export = session.data.tempoMap.getQuarterNotesPerMinute()
        else:
            export = None
        for func in self.quarterNotesPerMinuteChanged:
            func(export)
        callbacks._dataChanged()

    def _setPlaybackTicks(self): #Differs from the template because it has subdivisions and offset
        ppqn = cbox.Transport.status().pos_ppqn * session.data.subdivisions  - session.data.cachedOffsetInTicks
        status = playbackStatus()
        for func in self.setPlaybackTicks:
            func(ppqn, status)

    def _loopChanged(self, measurenumber, loopStart, loopEnd):
        export = measurenumber
        for func in self.loopChanged:
            func(export)

    def _loopMeasureFactorChanged(self):
        """Very atomic callback. Used only for one value: how many measures are in one loop"""
        export = session.data.loopMeasureFactor
        for func in self.loopMeasureFactorChanged:
            func(export)

    def _timeSignatureChanged(self):
        nr = session.data.howManyUnits
        typ = session.data.whatTypeOfUnit
        for func in self.timeSignatureChanged:
            func(nr, typ)

        ##All patterns and tracks need updates:
        for track in session.data.tracks:
            self._patternChanged(track)

        self._subdivisionsChanged() #update subdivisions. We do not include them in the score or track export on purpose.

        callbacks._dataChanged()

    def _subdivisionsChanged(self):
        """Subdivisions are tricky, therefore we keep them isolated in their own callback.
        You don't need to redraw anything if you don't want to. One recommendation is to
        draw every n step a little more important (bigger, different color).
        where n = subdivions

        We also place JACK BBT via tempoMap here because we need it every time the time sig
        changes (which calls _subdivisionsChanged) and if subdivisions change.
        """
        typ = baseDurationToTraditionalNumber[session.data.whatTypeOfUnit]
        nr = session.data.howManyUnits
        tradNr = int(nr / session.data.subdivisions)
        #JACK BBT for Timebase Master. No matter if we are master at the moment or not.
        if tradNr == nr / session.data.subdivisions:
            #Easier to read than the else case. Not possible with 9 Steps per Pattern in Groups of 2 because that is a 4.5/4 timesig.
            session.data.tempoMap.setTimeSignature(tradNr, typ)
        else:
            #Always possible, compared to first if case.
            session.data.tempoMap.setTimeSignature(nr, typ*session.data.subdivisions)

        export = session.data.subdivisions
        for func in self.subdivisionsChanged:
            func(export)
        callbacks._dataChanged()


    def _scoreChanged(self):
        """This includes the time signature as well, but is not send on a timesig change.
        A timesig change needs to update all tracks as playback as well as for the GUI
        so it is its own callback.
        Use this for fast and inexpensive updates like drawing a label or adjusting the
        GUI that shows your measure groups (a label each 8 measures or so)"""
        export = session.data.export()
        for func in self.scoreChanged:
            func(export)
        callbacks._dataChanged()

    def _exportCacheChanged(self, track):
        """Send the export cache for GUI caching reasons. Don't react by redrawing immediately!
        This is sent often, redundantly and more than you need.
        Example: If you only show one pattern at the same time use this to cache the data
        in all hidden patterns and redraw only if you change the active pattern.
        You can react to real changes in your active pattern by using patternChanged
        and stepChanged."""
        export = track.export()
        for func in self.exportCacheChanged:
            func(export)

    def _patternChanged(self, track):
        """each track has only one pattern. We can identify the pattern by track and vice versa.
        Don't use this to react to clicks on the pattern editor. Use stepChanged instead and
        keep book of your incremental updates.
        This is used for the whole pattern: timesig changes, invert, clean etc.
        """
        export = track.export()
        self._exportCacheChanged(track)
        for func in self.patternChanged:
            func(export)
        callbacks._dataChanged()

    def _stepChanged(self, track, stepDict):
        """A simple GUI will most like not listen to that callback since they
        already changed the step on their side. Only useful for parallel
        views.
        We do not export anything but just sent back the change we received
        as dict message."""
        self._exportCacheChanged(track)
        for func in self.stepChanged:
            func(stepDict)
        callbacks._dataChanged()

    def _removeStep(self, track, index, pitch, factor):
        """Opposite of _stepChanged"""
        self._exportCacheChanged(track)
        stepDict =  {"index": index,
                "factor": factor,
                "pitch": pitch,
                "velocity": 0, #it is off.
        }
        for func in self.removeStep:
            func(stepDict)
        callbacks._dataChanged()

    def _trackStructureChanged(self, track):
        """update one track structure. Does not export cbox.
        Also includes transposition """
        export = track.export()
        for func in self.trackStructureChanged:
            func(export)
        callbacks._dataChanged()

    def _trackMetaDataChanged(self, track):
        """a low cost function that should not trigger anything costly to redraw
        but some text and simple widgets."""
        export = track.export()
        for func in self.trackMetaDataChanged:
            func(export)
        callbacks._dataChanged()

    def _numberOfMeasuresChanged(self):
        """The whole song got longer or shorter.
        Number of measures is the same for all tracks."""
        export = session.data.export()
        for func in self.numberOfMeasuresChanged:
            func(export)
        callbacks._dataChanged()

    def _patternLengthMultiplicatorChanged(self, track):
        export = track.export()
        for func in self.patternLengthMultiplicatorChanged:
            func(export)
        self._patternChanged(track) #includes dataChanged
        self._subdivisionsChanged()
        callbacks._dataChanged()

    def _swingChanged(self):
        """Global for the whole song"""
        export = session.data.swing
        for func in self.swingChanged:
            func(export)

        for func in self.swingPercentChanged:
            func(_swingToPercent_Table[export])
        callbacks._dataChanged()

    def _currentTrackChanged(self, track):
        """The engine has no concept of a current track. This is a callback to sync different
        GUIs and midi controllers that use the system of a current track.
        Therefore this callback will never get called on engine changes.

        We send this once after load (always track 0) and then non-engine is responsible for calling
        the api function api.changeCurrentTrack(trackId)
        """
        export = track.export()
        for func in self.currentTrackChanged:
            func(export)
        #no data changed.





#Inject our derived Callbacks into the parent module
template.engine.api.callbacks = ClientCallbacks()
from template.engine.api import callbacks

_templateStartEngine = startEngine

def updatePlayback():
    #TODO: use template.sequencer.py internal updates instead
    cbox.Document.get_song().update_playback()

def startEngine(nsmClient, additionalData:dict={}):
    _templateStartEngine(nsmClient, additionalData) #loads save files or creates empty structure.

    session.inLoopMode = None # remember if we are in loop mode or not. Positive value is None or  a tuple with start and end

    #Activate apc mini controller
    apcMiniInput.start()
    apcMiniInput.setMidiInputActive(True) #MidiInput is just the general "activate processing"
    #Send the current track, to at least tell apcMini where we are.
    changeCurrentTrack(id(session.data.tracks[0])) # See callback docstring. This is purely for GUI and midi-controller convenience. No engine data is touched.

    #Send initial Callbacks to create the first GUI state.
    #The order of initial callbacks must not change to avoid GUI problems.
    #For example it is important that the tracks get created first and only then the number of measures
    logger.info("Sending initial callbacks to GUI and APCmini")
    callbacks._numberOfTracksChanged()
    callbacks._timeSignatureChanged()
    callbacks._numberOfMeasuresChanged()
    callbacks._subdivisionsChanged()
    callbacks._quarterNotesPerMinuteChanged()
    callbacks._loopMeasureFactorChanged()
    callbacks._swingChanged()

    for track in session.data.tracks:
        callbacks._trackMetaDataChanged(track) #for colors, scale and simpleNoteNames
        callbacks._patternLengthMultiplicatorChanged(track) #for colors, scale and simpleNoteNames

    session.data.buildAllTracks(buildSongDuration=True) #will set to max track length, we always have a song duration.

    updatePlayback()

    logger.info("Patroneo api startEngine complete")


def _loopOff():
    session.data.buildSongDuration() #no parameter removes the loop
    updatePlayback()
    session.inLoopMode = None
    callbacks._loopChanged(None, None, None)

def _loopNow():
    now = cbox.Transport.status().pos_ppqn
    _setLoop(now)

def _setLoop(loopMeasureAroundPpqn:int):
    """This function is used with context.
    The loopFactor, how many measures are looped, is saved value """
    if loopMeasureAroundPpqn < 0:
        _loopOff()
        return

    #loopMeasureAroundPpqn = max(0, loopMeasureAroundPpqn + session.data.cachedOffsetInTicks)
    loopMeasureAroundPpqn = loopMeasureAroundPpqn - session.data.cachedOffsetInTicks

    loopStart, loopEnd = session.data.buildSongDuration(loopMeasureAroundPpqn) #includes global tick offset
    session.data._lastLoopStart = loopStart

    updatePlayback()
    session.inLoopMode = (loopStart, loopEnd)

    assert loopStart <= loopMeasureAroundPpqn+session.data.cachedOffsetInTicks < loopEnd, (loopStart, loopMeasureAroundPpqn, loopEnd)
    if not playbackStatus():
        cbox.Transport.play()

    oneMeasureInTicks = (session.data.howManyUnits * session.data.whatTypeOfUnit) / session.data.subdivisions

    measurenumber, rest = divmod(loopStart-session.data.cachedOffsetInTicks, oneMeasureInTicks) #We substract the offset from the measure number because for a GUI it is still the visible measure number

    callbacks._loopChanged(int(measurenumber), loopStart, loopEnd)

def setLoopMeasureFactor(newValue:int):
    """How many measures are looped at once."""
    if newValue < 1:
        newValue = 1
    session.data.loopMeasureFactor = newValue
    callbacks._loopMeasureFactorChanged()
    if session.inLoopMode:
        _setLoop(session.data._lastLoopStart)

def toggleLoop():
    """Plays the current measure as loop.
    Current measure is where the playback cursor is

    session.inLoopMode is a tuple (start, end)
    """
    if session.inLoopMode:
        _loopOff()
    else:
        _loopNow()

def rewind():
    """template.toStart, but removes our loop"""
    _loopOff()
    toStart()

def seek(value):
    """override template one, which does not have a loop"""
    if value < 0:
        value = 0
    if session.inLoopMode and not session.inLoopMode[0] <= value < session.inLoopMode[1]: #if you seek outside the loop the loop will be destroyed.
        toggleLoop()

    value = max(0, value + session.data.cachedOffsetInTicks)
    cbox.Transport.seek_ppqn(value)


def seekMeasureLeft():
    """This skips one base measure, not the multiplicator one"""
    now = cbox.Transport.status().pos_ppqn
    oneMeasureInTicks = (session.data.howManyUnits * session.data.whatTypeOfUnit) / session.data.subdivisions
    seek(now - oneMeasureInTicks)

def seekMeasureRight():
    """This skips one base measure, not the multiplicator one"""
    now = cbox.Transport.status().pos_ppqn
    oneMeasureInTicks = (session.data.howManyUnits * session.data.whatTypeOfUnit) / session.data.subdivisions
    seek(now + oneMeasureInTicks)


def getGlobalOffset():
    """Return the current offsets in full measures + free tick value 3rd: Cached abolute tick value
    gets updated everytime the time signature changes or setGlobalOffset is called"""
    return session.data.globalOffsetMeasures, session.data.globalOffsetTicks, session.data.cachedOffsetInTicks


def setGlobalOffset(fullMeasures, absoluteTicks):
    session.history.register(lambda f=session.data.globalOffsetMeasures, t=session.data.globalOffsetTicks: setGlobalOffset(f,t), descriptionString="Global Rhythm Offset")
    session.data.globalOffsetMeasures = fullMeasures
    session.data.globalOffsetTicks = absoluteTicks
    session.data.buildAllTracks() #includes refreshing the tick offset cache
    updatePlayback()

##Score
def set_quarterNotesPerMinute(value):
    """Transport Master is set implicitly. If value == None Patroneo will switch into
    JackTransport Slave mode"""
    if session.data.tempoMap.isTransportMaster:
        oldValue = session.data.tempoMap.getQuarterNotesPerMinute()
    else:
        oldValue = None

    if oldValue == value: return # no change

    session.history.register(lambda v=oldValue: set_quarterNotesPerMinute(v), descriptionString="Tempo")

    if value is None:
        session.data.tempoMap.isTransportMaster = False #triggers rebuild
    elif value == "on":
        assert not session.data.tempoMap.isTransportMaster
        #keep old bpm value. 120 bpm is default.
        session.data.tempoMap.isTransportMaster = True #triggers rebuild
    else:
        assert value > 0
        session.data.tempoMap.setQuarterNotesPerMinute(value)
        session.data.tempoMap.isTransportMaster = True #triggers rebuild
    #Does not need track rebuilding
    updatePlayback()
    callbacks._quarterNotesPerMinuteChanged()


def set_whatTypeOfUnit(ticks):
    """Denominator of Time Signature = Each Group produces a Quarter """
    if session.data.whatTypeOfUnit == ticks: return #no change

    session.history.register(lambda v=session.data.whatTypeOfUnit: set_whatTypeOfUnit(v), descriptionString="Group Duration")
    session.data.whatTypeOfUnit = ticks
    session.data.buildAllTracks()
    if session.inLoopMode:
        _loopNow()
    updatePlayback()
    callbacks._timeSignatureChanged()

def set_howManyUnits(value):
    """Numerator of Time Signature = Steps per Pattern"""
    if session.data.howManyUnits == value: return #no change

    session.history.register(lambda v=session.data.howManyUnits: set_howManyUnits(v), descriptionString="Steps per Pattern")
    session.data.howManyUnits = value
    session.data.buildAllTracks()
    if session.inLoopMode:
        _loopNow()
    updatePlayback()
    callbacks._timeSignatureChanged()


def set_subdivisions(value):
    """In groups of 1, 2, 4"""
    if session.data.subdivisions == value: return #no change
    session.history.register(lambda v=session.data.subdivisions: set_subdivisions(v), descriptionString="Group Size")
    session.data.subdivisions = value
    session.data.buildAllTracks()
    if session.inLoopMode:
        _loopNow()
    updatePlayback()
    callbacks._subdivisionsChanged()

def convert_subdivisions(value, errorHandling):
    """"errorHandling can be fail, delete or merge"""
    oldValue = session.data.subdivisions
    result = session.data.convertSubdivisions(value, errorHandling)
    if result: #bool for success
        session.history.register(lambda v=oldValue: convert_subdivisions(v, "delete"), descriptionString="Convert Grouping") #the error handling = delete should not matter at all. We are always in a position where this is possible because we just converted to the current state from a valid one.
        session.data.buildAllTracks()
        updatePlayback()
        callbacks._timeSignatureChanged() #includes pattern changed for all tracks and subdivisions
        #for tr in session.data.tracks:
        #    callbacks._patternChanged(tr)
    else:
        callbacks._subdivisionsChanged() #to reset the GUI value back to the working one.
    if session.inLoopMode:
        _loopNow()
    return result


def set_swing(value:float):
    """A swing that feels natural is not linear. This function sets the absolute value
    between -0.5 and 0.5 but you most likely want to use setSwingPercent which has a non-linear
    mapping"""
    if value < -0.5 or value > 0.5:
        logger.warning(f"Swing can only be between -0.5 and 0.5, not {value}")
        return

    session.history.register(lambda v=session.data.swing: set_swing(v), descriptionString="Swing")
    session.data.swing = value
    session.data.buildAllTracks()
    updatePlayback()
    callbacks._swingChanged()


def setSwingPercent(value:int):
    """Give value between -100 and 100. 0 is "off" and default.
    It will be converted to a number between -0.5 and 0.5 behind the scenes. This is the value
    that gets saved.
    Our function will use a lookup-table to convert percentage in a musical way.

    That said, the GUI and the apcMini only go from 0 to 100, the negative range is ignored.

    The first 80% will be used for normal musical values. The other 20 for more extreme sounds.
    """
    if value < -100 or value > 100:
        logger.warning(f"Swing in percent can only be between -100 and +100, not {value}")
        return
    set_swing(_percentToSwing_Table[value]) #handles undo and callbacks


def set_numberOfMeasures(value):
    if session.data.numberOfMeasures == value:
        return

    session.history.register(lambda v=session.data.numberOfMeasures: set_numberOfMeasures(v), descriptionString="Measures per Track")
    session.data.numberOfMeasures = value
    session.data.buildSongDuration()
    updatePlayback()
    callbacks._numberOfMeasuresChanged()
    callbacks._scoreChanged() #redundant but cheap and convenient

def set_measuresPerGroup(value):
    if session.data.measuresPerGroup == value:
        return
    session.history.register(lambda v=session.data.measuresPerGroup: set_measuresPerGroup(v), descriptionString="Measures per Group")
    session.data.measuresPerGroup = value
    #No playback change
    callbacks._scoreChanged()

def changeTrackName(trackId, name):
    """The template gurantees a unique, sanitized name across tracks and groups"""
    track = session.data.trackById(trackId)
    if not track: return
    if not name.lower() in (gr.lower() for gr in getGroups()):
        session.history.register(lambda trId=trackId, v=track.sequencerInterface.name: changeTrackName(trId,v), descriptionString="Track Name")
        track.sequencerInterface.name = name #sanitizes on its own. Checks for duplicate tracks but not groups
    callbacks._trackMetaDataChanged(track)

def changeTrackColor(trackId, colorInHex):
    """Expects "#rrggbb"""
    track = session.data.trackById(trackId)
    if not track: return
    assert len(colorInHex) == 7, colorInHex
    session.history.register(lambda trId=trackId, v=track.color: changeTrackColor(trId,v), descriptionString="Track Color")
    track.color = colorInHex
    callbacks._trackMetaDataChanged(track)

def changeTrackMidiChannel(trackId, newChannel:int):
    """newChannel is 1-16, we convert here to internal format 0-15.
    Callbacks export data sends 1-16 again"""
    if newChannel < 1 or newChannel > 16:
        logger.warning(f"Midi Channel must be between 1-16 for this function, was: {newChannel}. Doing nothing.")
        return
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.midiChannel: changeTrackMidiChannel(trId,v), descriptionString="Track Midi Channel")
    track.midiChannel = newChannel-1
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._trackMetaDataChanged(track)
    #But not the actual callback because nothing in a gui needs redrawing. Everything looks the same.

def changeTrackStepDelayWrapAround(trackId, newState:bool):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.stepDelayWrapAround: changeTrackStepDelayWrapAround(trId,v), descriptionString="Track Step Delay Wrap-Around")
    track.stepDelayWrapAround = newState
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._trackMetaDataChanged(track)
    #But not the actual callback because nothing in a gui needs redrawing. Everything looks the same.

def changeTrackRepeatDiminishedPatternInItself(trackId, newState:bool):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.repeatDiminishedPatternInItself: changeTrackRepeatDiminishedPatternInItself(trId,v), descriptionString="Track Repeat Diminished Pattern in itself")
    track.repeatDiminishedPatternInItself = newState
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._trackMetaDataChanged(track)
    #But not the actual callback because nothing in a gui needs redrawing. Everything looks the same.

def changeCurrentTrack(trackId):
    """This is for communication between the GUI and APCmini controller. The engine has no concept
    of a current track."""
    track = session.data.trackById(trackId)
    assert track, trackId
    callbacks._currentTrackChanged(track)

def currentTrackBy(currentTrackId, value:int):
    """Convenience for the apcMiniController or a GUI that wants shortcuts.
    Only use +1 and -1 for value for stepping.
    We do NOT test for other values if the overshoot the session.data.tracks index!!
    We need to know what the current track is because the engine doesn't know it.

    Ignores invisible tracks, aka tracks in a GUI-folded group.
    """

    assert value in (-1, 1), value

    currentTrack = session.data.trackById(currentTrackId)
    assert currentTrack, currentTrackId
    onlyVisibleTracks = [track for track in session.data.tracks if track.visible]
    if not onlyVisibleTracks: return; #all tracks are hidden


    if not currentTrack in onlyVisibleTracks:
        changeCurrentTrack(id(onlyVisibleTracks[0]))
        return #an impossible situation.

    currentIndex = onlyVisibleTracks.index(currentTrack)

    if value == -1 and currentIndex == 0: return #already first track
    elif value == 1 and len(onlyVisibleTracks) == currentIndex+1: return #already last track

    newCurrentTrack = onlyVisibleTracks[currentIndex+value]
    changeCurrentTrack(id(newCurrentTrack))

def addTrack(scale=None):
    if scale:
        assert type(scale) == tuple
    track = session.data.addTrack(scale=scale)
    assert track
    trackId = id(track)
    session.history.register(lambda trId=trackId: deleteTrack(trId), descriptionString="Add Track")
    session.data.sortTracks() #in place sorting for groups
    callbacks._numberOfTracksChanged()
    return trackId

def createSiblingTrack(trackId): #aka clone track
    """Create a new track with scale, color and jack midi out the same as the given track.
    The jack midi out will be independent after creation, but connected to the same instrument
    (if any)"""
    track = session.data.trackById(trackId)
    if not track: return
    assert type(track.pattern.scale) == tuple
    newTrack = session.data.addTrack(name=track.sequencerInterface.name, scale=track.pattern.scale, color=track.color, simpleNoteNames=track.pattern.simpleNoteNames) #track name increments itself from "Track A" to "Track B" or "Track 1" -> "Track 2"
    newTrack.pattern.averageVelocity = track.pattern.averageVelocity
    newTrack.patternLengthMultiplicator = track.patternLengthMultiplicator
    newTrack.midiChannel = track.midiChannel

    if track.group:
        session.data.setGroup(newTrack, track.group) #includes session.data.buildAllTracks()
    else:
        jackConnections = cbox.JackIO.get_connected_ports(track.sequencerInterface.cboxPortName())
        for port in jackConnections:
            cbox.JackIO.port_connect(newTrack.sequencerInterface.cboxPortName(), port)

    #Move new track to neighbour the old one.
    oldIndex = session.data.tracks.index(track)
    newIndex = session.data.tracks.index(newTrack)
    newTrackAgain = session.data.tracks.pop(newIndex)
    assert newTrackAgain is newTrack
    session.data.tracks.insert(oldIndex+1, newTrackAgain)
    session.history.register(lambda trId=id(newTrackAgain): deleteTrack(trId), descriptionString="Clone Track")
    session.data.sortTracks() #in place sorting for groups
    callbacks._numberOfTracksChanged()
    return newTrack.export()

def _reinsertDeletedTrack(track, trackIndex):
    """For undo"""
    track.sequencerInterface.recreateThroughUndo()
    session.data.tracks.insert(trackIndex, track)
    session.history.register(lambda trId=id(track): deleteTrack(trId), descriptionString="Add deleted Track again")
    updatePlayback()
    callbacks._numberOfTracksChanged()

def deleteTrack(trackId):
    """ indirectly calls session.data.buildAllTracks() through group change.
    This is wasteful, but it acceptable. We let the code stay simple in exchange for redundant
    re-building of all tracks.
    """

    track = session.data.trackById(trackId)
    if not track: return
    oldIndex = session.data.tracks.index(track)

    with session.history.sequence("Delete Track"):
        setTrackGroup(trackId, "") #has it's own undo
        deletedTrack = session.data.deleteTrack(track)

        if not session.data.tracks: #always keep at least one track
            addTrack() #has it's own undo
            session.history.register(lambda tr=deletedTrack, pos=oldIndex: _reinsertDeletedTrack(tr, pos), descriptionString="Delete Track")
        else:
            session.history.register(lambda tr=deletedTrack, pos=oldIndex: _reinsertDeletedTrack(tr, pos), descriptionString="Delete Track")

    updatePlayback()

    callbacks._numberOfTracksChanged() #TODO: throws a console error "port not found". but that is not critical.


def moveTrack(trackId, newIndex):
    """index is 0 based.
    With groups involved free movevement is not allowed anymore.
    All tracks of a group have to be next to each other and have to be in that exact place
    in session.data.tracks , so that jack metadata port order works, groups or not.
    """
    track = session.data.trackById(trackId)
    if not track: return
    oldIndex = session.data.tracks.index(track)
    if not oldIndex == newIndex:
        session.history.register(lambda tr=trackId, pos=oldIndex: moveTrack(trackId, pos), descriptionString="Move Track")
        session.data.tracks.pop(oldIndex)
        session.data.tracks.insert(newIndex, track)
        session.data.sortTracks() #in place sorting for groups
        callbacks._numberOfTracksChanged()


def setTrackPatternLengthMultiplicator(trackId, newMultiplicator:int):
    if newMultiplicator < 1 or not isinstance(newMultiplicator, int):
        return #Invalid input

    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda tr=trackId, v=track.patternLengthMultiplicator: setTrackPatternLengthMultiplicator(trackId, v), descriptionString="Pattern Multiplier")
    track.patternLengthMultiplicator = newMultiplicator
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()

    #Order is important! Otherwise the GUI doesn't know that new empty steps need to exist to fill in.
    callbacks._patternLengthMultiplicatorChanged(track)
    callbacks._patternChanged(track)

#Track Groups
#Groups are dynamic. What groups exists and in which order is derived from the tracks themselves

def getGroups():
    """
    Returns an iterator of strings in order of the tracks.
    Will return only existing groups, that contain at least one track"""
    return session.data.groups.keys()

def setTrackGroup(trackId, groupName:str):
    """A not yet existing groupName will create that.
    Set to empty string to create a standalone track"""
    track = session.data.trackById(trackId)
    if not track: return
    groupName = ''.join(ch for ch in groupName if ch.isalnum()) #sanitize
    groupName = " ".join(groupName.split()) #remove double spaces
    if not track.group == groupName:
        if not groupName.lower() in (track.sequencerInterface.name.lower() for track in session.data.tracks):
            session.history.register(lambda tr=trackId, v=track.group: setTrackGroup(trackId, v), descriptionString="Track Group")
            session.data.setGroup(track, groupName) #includes session.data.buildAllTracks(). This is wasteful, but it acceptable. We let the code stay simple in exchange for redundant re-building of all tracks.
            updatePlayback()
        callbacks._numberOfTracksChanged()

def moveGroup(groupName:str, newIndex:int):
    """"
    index is 0 based.
    newIndex is like a track index. But instead of moving a single track we move all tracks
    of one group to this position"""

    #find tracks with that group.
    #We assume they are all next to each other, because that is how session.data auto-sorts tracks
    groupMembers = [track for track in session.data.tracks if track.group == groupName]
    firstGroupTrack = groupMembers[0]
    firstGroupTrackIndex = session.data.tracks.index(firstGroupTrack)

    if firstGroupTrackIndex == newIndex:
        return

    session.history.register(lambda gr=groupName, pos=firstGroupTrackIndex: moveGroup(gr, pos), descriptionString="Move Group")
    for offset, track in enumerate(groupMembers):
        #We can't check and assert indices here because the list changes under our nose.
        #assert firstGroupTrackIndex + offset == session.data.tracks.index(track), (firstGroupTrackIndex, offset, session.data.tracks.index(track), track.sequencerInterface.name)
        #popTr = session.data.tracks.pop(firstGroupTrackIndex + offset)
        popTr = session.data.tracks.pop(session.data.tracks.index(track))
        #assert track is popTr, (track, track.sequencerInterface.name, popTr, popTr.sequencerInterface.name )
        session.data.tracks.insert(newIndex+offset, track)
    session.data.sortTracks() #in place sorting for groups
    callbacks._numberOfTracksChanged()


def setGroupVisible(groupName:str, force:bool=None):
    """A convenience function for the gui. just a flag that gets saved and loaded and changes
    are reported via callback

    Hides all tracks belonging to that track in reality. But we offer no way to hide a non-group
    track.

    Calling without the force parameter to True/False toggles visibility.
    """
    groupMembers = [track for track in session.data.tracks if track.group == groupName]
    for track in groupMembers:
        if track.group == groupName:
            if not force is None:
                track.visible = bool(force)
            else:
                track.visible = not track.visible
    #no need to update playback
    session.data.sortTracks() #in place sorting for groups
    callbacks._numberOfTracksChanged()




#Track Switches
#Aka measures
def _setTrackStructure(trackId, structure, undoMessage):
    """For undo. Used by all functions as entry point, then calls itself for undo/redo.
    structure is a set of integers which we can copy with .copy()"""
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda tr=trackId, v=track.structure.copy(), msg=undoMessage: _setTrackStructure(trackId, v, msg), descriptionString=undoMessage)

    track.structure = structure #restore data
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)


def _removeMeasureModificationsWithUndo(trackId, position):
    """Must be run in a history sequence context manager!"""
    track = session.data.trackById(trackId)
    if position in track.whichPatternsAreScaleTransposed:
        setSwitchScaleTranspose(trackId, position, 0) #set to default value. track export will remove the value from the data
    if position in track.whichPatternsAreHalftoneTransposed:
        setSwitchHalftoneTranspose(trackId, position, 0) #set to default value. track export will remove the value from the data
    if position in track.whichPatternsAreStepDelayed:
        setSwitchStepDelay(trackId, position, 0) #set to default value. track export will remove the value from the data
    if position in track.whichPatternsHaveAugmentationFactor:
        setSwitchAugmentationsFactor(trackId, position, 1.0)  #set to default value. track export will remove the value from the data


def setSwitches(trackId, setOfPositions, newBool):
    """Used in the GUI to select multiple switches in a row by dragging the mouse"""
    track = session.data.trackById(trackId)
    if not track: return

    with session.history.sequence("Set Measures"):
        session.history.register(lambda tr=trackId, v=track.structure.copy(): _setTrackStructure(trackId, v, "Set Measures"), descriptionString="Set Measures")

        if newBool:
            track.structure = track.structure.union(setOfPositions) #merge: add setOfPositions to the existing one
        else:
            track.structure = track.structure.difference(setOfPositions) #replace: remove everything from setOfPositions that is in the existing one, ignore entries from setOfPositions not in existing.

        for position in setOfPositions:
            _removeMeasureModificationsWithUndo(trackId, position)

    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def setSwitch(trackId, position, newBool):
    """e.g. for GUI Single click operations. Switch on and off a measure"""

    track = session.data.trackById(trackId)
    if not track: return

    with session.history.sequence("Set Measures"):
        session.history.register(lambda trId=trackId, v=track.structure.copy(): _setTrackStructure(trId, v, "Set Measures"), descriptionString="Set Measures")

        if newBool:
            if position in track.structure: return
            track.structure.add(position)
        else:
            if not position in track.structure: return
            track.structure.remove(position)

        _removeMeasureModificationsWithUndo(trackId, position)

    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)
    return True

def trackInvertSwitches(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.structure.copy(): _setTrackStructure(trId, v, "Invert Measures"), descriptionString="Invert Measures")
    """
    if track.structure:
        new = set(i for i in range(max(track.structure)))
        track.structure = new.difference(track.structure)
    else:
        track.structure = set(i for i in range(session.data.numberOfMeasures))
    """
    new = set(i for i in range(session.data.numberOfMeasures))
    track.structure = new.difference(track.structure)
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def trackOffAllSwitches(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.structure.copy(): _setTrackStructure(trId, v, "Track Measures Off"), descriptionString="Track Measures Off")
    track.structure = set()
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def trackOnAllSwitches(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.structure.copy(): _setTrackStructure(trId, v, "Track Measures On"), descriptionString="Track Measures On")
    track.structure = set(i for i in range(session.data.numberOfMeasures))
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def trackMergeCopyFrom(sourceTrackId, targetTrackId):
    if not sourceTrackId == targetTrackId:
        sourceTrack = session.data.trackById(sourceTrackId)
        targetTrack = session.data.trackById(targetTrackId)
        session.history.register(lambda trId=id(targetTrack), v=targetTrack.structure.copy(): _setTrackStructure(trId, v, "Copy Measures"), descriptionString="Copy Measures")
        targetTrack.structure = targetTrack.structure.union(sourceTrack.structure)
        targetTrack.whichPatternsAreScaleTransposed.update(sourceTrack.whichPatternsAreScaleTransposed)
        targetTrack.whichPatternsAreHalftoneTransposed.update(sourceTrack.whichPatternsAreHalftoneTransposed)
        targetTrack.whichPatternsAreStepDelayed.update(sourceTrack.whichPatternsAreStepDelayed)
        targetTrack.whichPatternsHaveAugmentationFactor.update(sourceTrack.whichPatternsHaveAugmentationFactor)
        targetTrack.buildTrack()
        updatePlayback()
        callbacks._trackStructureChanged(targetTrack)

def trackPatternReplaceFrom(sourceTrackId, targetTrackId):
    if not sourceTrackId == targetTrackId:
        sourceTrack = session.data.trackById(sourceTrackId)
        targetTrack = session.data.trackById(targetTrackId)
        session.history.register(lambda trId=id(targetTrack), v=targetTrack.structure.copy(): _setTrackStructure(trId, v, "Replace Measures"), descriptionString="Replace Measures")

        copyPattern = sourceTrack.pattern.copy(newParentTrack = targetTrack)
        targetTrack.pattern = copyPattern

        targetTrack.buildTrack()
        updatePlayback()
        callbacks._patternChanged(targetTrack)

#Transpositions and Modal Shifts
#StepDelay and AugmentationFactor

def _setSwitchesScaleTranspose(trackId, whichPatternsAreScaleTransposed):
    """For undo. Used by all functions as entry point, then calls itself for undo/redo.
    whichPatternsAreScaleTransposed is a dicts of int:int which we can copy with .copy()"""
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda tr=trackId, v=track.whichPatternsAreScaleTransposed.copy(): _setSwitchesScaleTranspose(trackId, v), descriptionString="Set Modal Shift")

    track.whichPatternsAreScaleTransposed = whichPatternsAreScaleTransposed #restore data
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)


def setSwitchScaleTranspose(trackId, position:int, transpose:int):
    """Scale transposition is flipped. lower value means higher pitch.
    Default value is 0."""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda tr=trackId, v=track.whichPatternsAreScaleTransposed.copy(): _setSwitchesScaleTranspose(trackId, v), descriptionString="Set Modal Shift")
    track.whichPatternsAreScaleTransposed[position] = transpose
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)
    return True


def _setSwitchHalftoneTranspose(trackId, whichPatternsAreHalftoneTransposed):
    """For undo. Used by all functions as entry point, then calls itself for undo/redo.
    whichPatternsAreScaleTransposed is a dicts of int:int which we can copy with .copy()"""
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda tr=trackId, v=track.whichPatternsAreHalftoneTransposed.copy(): _setSwitchHalftoneTranspose(trackId, v), descriptionString="Set Half Tone Shift")

    track.whichPatternsAreHalftoneTransposed = whichPatternsAreHalftoneTransposed #restore data

    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)


def setSwitchHalftoneTranspose(trackId, position:int, transpose:int):
    """Halftone transposition is not flipped. Higher value means higher pitch
    Default value is 0.
    """
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda tr=trackId, v=track.whichPatternsAreHalftoneTransposed.copy(): _setSwitchHalftoneTranspose(trackId, v), descriptionString="Set Half Tone Shift")
    track.whichPatternsAreHalftoneTransposed[position] = transpose
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)
    return True

def _setSwitchStepDelay(trackId, whichPatternsAreStepDelayed):
    """For undo. Used by all functions as entry point, then calls itself for undo/redo.
    whichPatternsAreStepDelayed is a dicts of int:int which we can copy with .copy()"""
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda tr=trackId, v=track.whichPatternsAreStepDelayed.copy(): _setSwitchStepDelay(trackId, v), descriptionString="Set Step Delay")

    track.whichPatternsAreStepDelayed = whichPatternsAreStepDelayed #restore data
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def setSwitchStepDelay(trackId, position:int, delay:int):
    """Public entry function for _setSwitchStepDelay.
    Default value is 0"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda tr=trackId, v=track.whichPatternsAreStepDelayed.copy(): _setSwitchStepDelay(trackId, v), descriptionString="Set Step Delay")
    track.whichPatternsAreStepDelayed[position] = delay
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)
    return True


def _setSwitchAugmentationsFactor(trackId, whichPatternsHaveAugmentationFactor):
    """For undo. Used by all functions as entry point, then calls itself for undo/redo.
    whichPatternsHaveAugmentationFactor is a dicts of int:float which we can copy with .copy()"""
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda tr=trackId, v=track.whichPatternsHaveAugmentationFactor.copy(): _setSwitchAugmentationsFactor(trackId, v), descriptionString="Set Augmentation Factor")

    track.whichPatternsHaveAugmentationFactor = whichPatternsHaveAugmentationFactor #restore data
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)

def setSwitchAugmentationsFactor(trackId, position:int, factor:float):
    """Public entry function for _setSwitchAugmentationsFactor.
    Default value is 1.0"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda tr=trackId, v=track.whichPatternsHaveAugmentationFactor.copy(): _setSwitchAugmentationsFactor(trackId, v), descriptionString="Set Augmentation Factor")
    track.whichPatternsHaveAugmentationFactor[position] = factor
    track.buildTrack()
    updatePlayback()
    callbacks._trackStructureChanged(track)
    return True


def _registerHistoryWholeTrackSwitches(track):
    """This is used by insertSilence, clearSwitchGroupModifications,
    exchangeSwitchGroupWithGroupToTheRight etc.

    It assumes that it runs inside this context:
        with session.history.sequence("asdasd"):
    """
    trackId = id(track)
    session.history.register(lambda trId=trackId, v=track.patternLengthMultiplicator: setTrackPatternLengthMultiplicator(trId, v), descriptionString="Pattern Multiplier")
    session.history.register(lambda trId=trackId, v=track.structure.copy(): _setTrackStructure(trId, v, "Change Group"), descriptionString="Change Group")
    session.history.register(lambda trId=trackId, v=track.whichPatternsAreScaleTransposed.copy(): _setSwitchesScaleTranspose(trId, v), descriptionString="Set Modal Shift")
    session.history.register(lambda trId=trackId, v=track.whichPatternsAreHalftoneTransposed.copy(): _setSwitchHalftoneTranspose(trId, v), descriptionString="Set Half Tone Shift")
    session.history.register(lambda trId=trackId, v=track.whichPatternsAreStepDelayed.copy(): _setSwitchStepDelay(trId, v), descriptionString="Set Step Delay")
    session.history.register(lambda trId=trackId, v=track.whichPatternsHaveAugmentationFactor.copy(): _setSwitchAugmentationsFactor(trId, v), descriptionString="Set Augmentation Factor")

def insertSilence(howMany:int, beforeMeasureNumber:int):
    """Insert empty measures into all tracks.
    Parameters are un-multiplied."""

    #In each track shift every switch to the right if it is before the dividing measure number
    #Keep the concept of a "group" even if there are multiplicators in the track.
    #If you insert 4 normal measures then a multiplicator-track of two gets only 2 new ones.
    with session.history.sequence("Insert/Duplicate Group"):  #this actually handles duplicateSwitchGroup undo as well!!!
        for track in session.data.tracks:
            _registerHistoryWholeTrackSwitches(track)

            thisTrackWhere = beforeMeasureNumber // track.patternLengthMultiplicator #integer division
            thisTrackHowMany = howMany // track.patternLengthMultiplicator #integer division
            track.structure = set( (switch + thisTrackHowMany if switch >= thisTrackWhere else switch) for switch in track.structure )
            track.whichPatternsAreScaleTransposed = { (k+thisTrackHowMany if k >= thisTrackWhere else k):v for k,v in track.whichPatternsAreScaleTransposed.items() }
            track.whichPatternsAreHalftoneTransposed = { (k+thisTrackHowMany if k >= thisTrackWhere else k):v for k,v in track.whichPatternsAreHalftoneTransposed.items() }
            track.whichPatternsAreStepDelayed = { (k+thisTrackHowMany if k >= thisTrackWhere else k):v for k,v in track.whichPatternsAreStepDelayed.items() }
            track.whichPatternsHaveAugmentationFactor = { (k+thisTrackHowMany if k >= thisTrackWhere else k):v for k,v in track.whichPatternsHaveAugmentationFactor.items() }

            callbacks._trackStructureChanged(track)

    callbacks._dataChanged() #register undo
    session.data.buildAllTracks()
    updatePlayback()

def duplicateSwitchGroup(startMeasureForGroup:int, endMeasureExclusive:int):
    """startMeasureForGroup and endMeasureExclusive are in the global, un-multiplied measure counting
    format."""

    groupSize = endMeasureExclusive-startMeasureForGroup

    #Undo:   InsertSilence has a complete undo already registered. We chose a neutral undo description so it handles both duplicate and insert silence.
    insertSilence(groupSize, endMeasureExclusive) #insert silence handles multiplicator-tracks on its own
    for track in session.data.tracks:
        thisTrackStartMeasure = startMeasureForGroup // track.patternLengthMultiplicator #integer division
        thisTrackEndMeasure = endMeasureExclusive // track.patternLengthMultiplicator
        thisGroupSize = groupSize // track.patternLengthMultiplicator

        for switch in range(thisTrackStartMeasure+thisGroupSize, thisTrackEndMeasure+thisGroupSize): #One group after the given one.
            if switch-thisGroupSize in track.structure:
                track.structure.add(switch)
            if switch-thisGroupSize in track.whichPatternsAreScaleTransposed:
                track.whichPatternsAreScaleTransposed[switch] = track.whichPatternsAreScaleTransposed[switch-thisGroupSize]
            if switch-thisGroupSize in track.whichPatternsAreHalftoneTransposed:
                track.whichPatternsAreHalftoneTransposed[switch] = track.whichPatternsAreHalftoneTransposed[switch-thisGroupSize]
            if switch-thisGroupSize in track.whichPatternsAreStepDelayed:
                track.whichPatternsAreStepDelayed[switch] = track.whichPatternsAreStepDelayed[switch-thisGroupSize]
            if switch-thisGroupSize in track.whichPatternsHaveAugmentationFactor:
                track.whichPatternsHaveAugmentationFactor[switch] = track.whichPatternsHaveAugmentationFactor[switch-thisGroupSize]
            callbacks._trackStructureChanged(track)

    session.data.buildAllTracks()
    updatePlayback()


def exchangeSwitchGroupWithGroupToTheRight(startMeasureForGroup:int, endMeasureExclusive:int):
    """The group is defined by the given measure range. The group right of it has the same dimensions.
    In a GUI you can use that to move groups left and right. We only supply the "switch with right"
    variant (and not left) because that is easier to comprehend.
    """

    with session.history.sequence("Exchange Group Order"):
        for track in session.data.tracks:
            _registerHistoryWholeTrackSwitches(track)

            thisTrackStartMeasure = startMeasureForGroup // track.patternLengthMultiplicator #integer division
            thisTrackEndMeasure = endMeasureExclusive // track.patternLengthMultiplicator
            groupSize = thisTrackEndMeasure - thisTrackStartMeasure

            assert thisTrackStartMeasure + groupSize == thisTrackEndMeasure, (thisTrackStartMeasure, groupSize, thisTrackEndMeasure)

            tempStructure = set() #integers
            tempScaleTransposed  = dict()  #position:integers
            tempHalfToneTransposed = dict() #position:integers
            tempStepDelayed = dict() #position:integers
            tempAugmentedFactor = dict() #position:floats

            #Remember for later testing
            lenStructure = len(track.structure)
            lenHalfToneTransposed = len(track.whichPatternsAreHalftoneTransposed.keys())
            lenScaleTransposed = len(track.whichPatternsAreScaleTransposed.keys())
            lenStepDelayed = len(track.whichPatternsAreStepDelayed.keys())
            lenAugmentedFactor = len(track.whichPatternsHaveAugmentationFactor.keys())

            #First move right group into a temporary buffer to have it out of the way
            for switch in range(thisTrackStartMeasure+groupSize, thisTrackEndMeasure+groupSize): #switch is a number
                if switch in track.structure:
                    tempStructure.add(switch)
                    track.structure.remove(switch)
                if switch in track.whichPatternsAreScaleTransposed:
                    tempScaleTransposed[switch] = track.whichPatternsAreScaleTransposed[switch]
                    del track.whichPatternsAreScaleTransposed[switch]
                if switch in track.whichPatternsAreHalftoneTransposed:
                    tempHalfToneTransposed[switch] = track.whichPatternsAreHalftoneTransposed[switch]
                    del track.whichPatternsAreHalftoneTransposed[switch]
                if switch in track.whichPatternsAreStepDelayed:
                    tempStepDelayed[switch] = track.whichPatternsAreStepDelayed[switch]
                    del track.whichPatternsAreStepDelayed[switch]
                if switch in track.whichPatternsHaveAugmentationFactor:
                    tempAugmentedFactor[switch] = track.whichPatternsHaveAugmentationFactor[switch]
                    del track.whichPatternsHaveAugmentationFactor[switch]

            #Now move current group to the right, which is now empty.
            for switch in range(thisTrackStartMeasure, thisTrackEndMeasure): #switch is a number
                if switch in track.structure:
                    track.structure.add(switch + groupSize)
                    track.structure.remove(switch)
                if switch in track.whichPatternsAreScaleTransposed:
                    track.whichPatternsAreScaleTransposed[switch+groupSize] = track.whichPatternsAreScaleTransposed[switch]
                    del track.whichPatternsAreScaleTransposed[switch]
                if switch in track.whichPatternsAreHalftoneTransposed:
                    track.whichPatternsAreHalftoneTransposed[switch+groupSize] = track.whichPatternsAreHalftoneTransposed[switch]
                    del track.whichPatternsAreHalftoneTransposed[switch]
                if switch in track.whichPatternsAreStepDelayed:
                    track.whichPatternsAreStepDelayed[switch+groupSize] = track.whichPatternsAreStepDelayed[switch]
                    del track.whichPatternsAreStepDelayed[switch]
                if switch in track.whichPatternsHaveAugmentationFactor:
                    track.whichPatternsHaveAugmentationFactor[switch+groupSize] = track.whichPatternsHaveAugmentationFactor[switch]
                    del track.whichPatternsHaveAugmentationFactor[switch]



            #Move old right-group into its new place
            for sw in tempStructure:
                track.structure.add(sw-groupSize)
            for stPos, stVal in tempScaleTransposed.items():
                track.whichPatternsAreScaleTransposed[stPos-groupSize] = stVal
            for htPos, htVal in tempHalfToneTransposed.items():
                track.whichPatternsAreHalftoneTransposed[htPos-groupSize] = htVal
            for sDPos, sDVal in tempStepDelayed.items():
                track.whichPatternsAreStepDelayed[sDPos-groupSize] = sDVal
            for aFPos, aFVal in tempAugmentedFactor.items():
                track.whichPatternsHaveAugmentationFactor[aFPos-groupSize] = aFVal

            callbacks._trackStructureChanged(track)

            #Do some tests
            assert lenStructure == len(track.structure), (lenStructure, len(track.structure))
            assert lenScaleTransposed == len(track.whichPatternsAreScaleTransposed.keys()), (lenScaleTransposed, len(track.whichPatternsAreScaleTransposed.keys()))
            assert lenHalfToneTransposed == len(track.whichPatternsAreHalftoneTransposed.keys()), (lenHalfToneTransposed, len(track.whichPatternsAreHalftoneTransposed.keys()))
            assert lenStepDelayed == len(track.whichPatternsAreStepDelayed.keys()), (lenStepDelayed, len(track.whichPatternsAreStepDelayed.keys()))
            assert lenAugmentedFactor == len(track.whichPatternsHaveAugmentationFactor.keys()), (lenAugmentedFactor, len(track.whichPatternsHaveAugmentationFactor.keys()))

    callbacks._dataChanged() #register undo
    session.data.buildAllTracks()
    updatePlayback()

def clearSwitchGroupModifications(startMeasureForGroup:int, endMeasureExclusive:int):
    """startMeasureForGroup and endMeasureExclusive are in the global, un-multiplied measure counting
    format."""
    with session.history.sequence("Clear all Group Transpositions"):
        for track in session.data.tracks:
            _registerHistoryWholeTrackSwitches(track)
            thisTrackStartMeasure = startMeasureForGroup // track.patternLengthMultiplicator #integer division
            thisTrackEndMeasure = endMeasureExclusive // track.patternLengthMultiplicator
            for switch in range(thisTrackStartMeasure, thisTrackEndMeasure):
                if switch in track.whichPatternsAreScaleTransposed:
                    del track.whichPatternsAreScaleTransposed[switch]
                if switch in track.whichPatternsAreHalftoneTransposed:
                    del track.whichPatternsAreHalftoneTransposed[switch]
                if switch in track.whichPatternsAreStepDelayed:
                    del track.whichPatternsAreStepDelayed[switch]
                if switch in track.whichPatternsHaveAugmentationFactor:
                    del track.whichPatternsHaveAugmentationFactor[switch]
            callbacks._trackStructureChanged(track)
    callbacks._dataChanged() #register undo
    session.data.buildAllTracks()
    updatePlayback()

def deleteSwitches(howMany, fromMeasureNumber):
    """Parameters are un-multiplied measures."""

    with session.history.sequence("Delete whole Group"):
        for track in session.data.tracks:
            _registerHistoryWholeTrackSwitches(track)

            thisTrackHowMany = howMany // track.patternLengthMultiplicator #integer division
            thisTrackWhere = fromMeasureNumber // track.patternLengthMultiplicator #integer division

            new_structure = set()

            for switch in track.structure:
                if switch < thisTrackWhere:
                    new_structure.add(switch)
                elif switch >= thisTrackWhere+thisTrackHowMany: #like a text editor let gravitate left into the hole left by the deleted range
                    new_structure.add(switch-thisTrackHowMany)
                #else: #discard all in range to delete
            track.structure = new_structure

            new_scaleTransposed = dict()
            for k,v in track.whichPatternsAreScaleTransposed.items():
                if k < thisTrackWhere:
                    new_scaleTransposed[k] = v
                elif k >= thisTrackWhere+thisTrackHowMany: #like a text editor let gravitate left into the hole left by the deleted range
                    new_scaleTransposed[k-thisTrackHowMany] = v
                #else: #discard all in range to delete
            track.whichPatternsAreScaleTransposed = new_scaleTransposed

            new_halftoneTransposed = dict()
            for k,v in track.whichPatternsAreHalftoneTransposed.items():
                if k < thisTrackWhere:
                    new_halftoneTransposed[k] = v
                elif k >= thisTrackWhere+thisTrackHowMany: #like a text editor let gravitate left into the hole left by the deleted range
                    new_halftoneTransposed[k-thisTrackHowMany] = v
                #else: #discard all in range to delete
            track.whichPatternsAreHalftoneTransposed = new_halftoneTransposed

            new_stepDelayed = dict()
            for k,v in track.whichPatternsAreStepDelayed.items():
                if k < thisTrackWhere:
                    new_stepDelayed[k] = v
                elif k >= thisTrackWhere+thisTrackHowMany: #like a text editor let gravitate left into the hole left by the deleted range
                    new_stepDelayed[k-thisTrackHowMany] = v
                #else: #discard all in range to delete
            track.whichPatternsAreStepDelayed = new_stepDelayed

            new_augmentFactor = dict()
            for k,v in track.whichPatternsHaveAugmentationFactor.items():
                if k < thisTrackWhere:
                    new_augmentFactor[k] = v
                elif k >= thisTrackWhere+thisTrackHowMany: #like a text editor let gravitate left into the hole left by the deleted range
                    new_augmentFactor[k-thisTrackHowMany] = v
                #else: #discard all in range to delete
            track.whichPatternsHaveAugmentationFactor = new_augmentFactor


            callbacks._trackStructureChanged(track)

    callbacks._dataChanged() #register undo
    session.data.buildAllTracks()
    updatePlayback()

#Pattern Steps

def setPattern(trackId, patternList, undoMessage):
    """Change the whole pattern, send a callback with the whole pattern.
    This is also the main undo/redo function.

    It is rarely used directly by the GUI, if at all. Normal changes are atomic.
    """
    track = session.data.trackById(trackId)
    if not track: return

    session.history.register(lambda trId=trackId, v=track.pattern.copyData(), msg=undoMessage: setPattern(trId, v, msg), descriptionString=undoMessage)

    track.pattern.data = patternList
    track.pattern.buildExportCache()
    track.buildTrack()
    callbacks._patternChanged(track)

def getAverageVelocity(trackId):
    """If a GUI wants to add a new note and choose a sensible velocity it can use this function"""
    return session.data.trackById(trackId).pattern.averageVelocity

def setStep(trackId, stepExportDict):
    """This is an atomic operation that only sets one switch and
    only sends that switch back via callback. A simple GUI
    will most like not listen to that callback since they
    already changed the step on their side. Only useful for parallel
    views.

    format:  {'index': 0, 'pitch': 7, 'factor': 1, 'velocity': 90}

    This is also for velocity!

    This function checks if the new step is within the limits of the current sounding pattern
    and will prevent changes or additions outside the current limits.
    """

    track = session.data.trackById(trackId)
    if not track: return

    #index is from 0 and howManyUnits*Multp is from 1 (because it is length), so we just need <, and not <=.
    inRange = stepExportDict["index"] < session.data.howManyUnits*track.patternLengthMultiplicator and stepExportDict["pitch"] < track.pattern.numberOfSteps
    if not inRange: return

    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Change Step"), descriptionString="Change Step")
    oldNote = track.pattern.stepByIndexAndPitch(index=stepExportDict["index"], pitch=stepExportDict["pitch"])
    if oldNote:  #modify existing note
        oldNoteIndex = track.pattern.data.index(oldNote)
        track.pattern.data.remove(oldNote)
        track.pattern.data.insert(oldNoteIndex, stepExportDict) #for what its worth, insert at the old place. It doesn't really matter though.
    else: #new note
        track.pattern.data.append(stepExportDict)

    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._stepChanged(track, stepExportDict)

def removeStep(trackId, index, pitch):
    """Reverse of setStep. e.g. GUI-Click on an existing step."""
    track = session.data.trackById(trackId)
    if not track: return

    inRange = index < session.data.howManyUnits*track.patternLengthMultiplicator and pitch < track.pattern.numberOfSteps
    if not inRange: return

    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Remove Step"), descriptionString="Remove Step")
    oldNote = track.pattern.stepByIndexAndPitch(index, pitch)
    track.pattern.data.remove(oldNote)
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._removeStep(track, index, pitch, oldNote["factor"])

def toggleStep(trackId, index, pitch, factor=1, velocity=None):
    """Checks the current state of a step and decides if on or off.
    Toggled Notes have average velocity and factor 1. If you need more fine control use setStep
    directly"""
    track = session.data.trackById(trackId)
    if not track: return
    maybeNote = track.pattern.stepByIndexAndPitch(index, pitch)
    if maybeNote is None:
        if velocity is None:
            velocity = getAverageVelocity(trackId)
        setStep(trackId, {'index': index, 'pitch': pitch, 'factor': factor, 'velocity': velocity})
    else:
        removeStep(trackId, index, pitch)


def setScale(trackId, scale, callback = True):
    """Expects a scale list or tuple from lowest index to highest.
    Actual pitches don't matter."""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.scale[:]: setScale(trId, v, callback=True), descriptionString="Set Scale")
    track.pattern.scale = scale #tuple, or list if oversight in json loading :)
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    if callback:
        callbacks._trackMetaDataChanged(track)

def setSimpleNoteNames(trackId, simpleNoteNames):
    """note names is a list of strings with length 128. One name for each midi note.
    It is saved to file"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.simpleNoteNames[:]: setSimpleNoteNames(trId, v), descriptionString="Note Names")
    track.pattern.simpleNoteNames = simpleNoteNames #list of strings
    callbacks._trackMetaDataChanged(track)

def transposeHalftoneSteps(trackId, steps):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.scale[:]: setScale(trId, v, callback=True), descriptionString="Transpose Scale")
    track.pattern.scale = [midipitch+steps for midipitch in track.pattern.scale]
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._trackMetaDataChanged(track)

def patternInvertSteps(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Invert Steps"), descriptionString="Invert Steps")
    track.pattern.invert()
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternOnAllSteps(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "All Steps On"), descriptionString="All Steps On")
    track.pattern.fill()
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternOffAllSteps(trackId):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "All Steps Off"), descriptionString="All Steps Off")
    track.pattern.empty()
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternInvertRow(trackId, pitchindex):
    """Pitchindex is the row"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Invert Row"), descriptionString="Invert Row")
    track.pattern.invertRow(pitchindex)
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternClearRow(trackId, pitchindex):
    """Pitchindex is the row.
    Index is the column"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Clear Row"), descriptionString="Clear Row")
    track.pattern.clearRow(pitchindex)
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternRowRepeatFromStep(trackId, pitchindex, index):
    """Pitchindex is the row.
    Index is the column"""
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Fill Row with Repeat"), descriptionString="Fill Row with Repeat")
    track.pattern.repeatFromStep(pitchindex, index)
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def patternRowChangeVelocity(trackId, pitchindex, delta):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Change Row Velocity"), descriptionString="Change Row Velocity")
    for note in track.pattern.getRow(pitchindex):
        new = note["velocity"] + delta
        note["velocity"] = min(max(new,0), 127)

    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)


major = [0, 2, 4, 5, 7, 9, 11] #this is sorted by pitch, lowest to highest. Patroneo works in reverse order to accomodate the row/column approach of a grid. We reverse in setScaleToKeyword
schemesDict = {
    #this if sorted by pitch, lowest to highest. Patroneo works in reverse order to accomodate the row/column approach of a grid. We reverse in setScaleToKeyword
    #The lowest/first pitch is always 0 because it is just the given root note.
    "Major": [0,0,0,0,0,0,0],
    "Minor": [0,0,-1,0,0,-1,-1],
    "Dorian": [0,0,-1,0,0,0,-1],
    "Phrygian": [0,-1,-1,0,0,-1,-1],
    "Lydian": [0,0,0,+1,0,0,0],
    "Mixolydian": [0,0,0,0,0,0,-1],
    "Locrian": [0,-1,-1,0,-1,-1,-1],
    #"Blues": [0,-2,-1,0,-1,-2,-1],  #blues is a special case. It has less notes than we offer.
    "Blues": [0, +1, +1, +1, 0, +2, +1], #broden. Needs double octave in the middle. better than completely wrong.
    "Hollywood": [0,0,0,0,0,-1,-1], #The "Hollywood"-Scale. Stargate, Lord of the Rings etc.
    "Chromatic": [0,-1,-2,-2,-3,-4,-5,-5,-6, -7, -7, -8, -9, -10, -10], #crude... also broken > 2 octaves
}
major.reverse()
for l in schemesDict.values():
    l.reverse()

#Ordered version
schemes = [
   "Major",
    "Minor",
    "Dorian",
    "Phrygian",
    "Lydian",
    "Mixolydian",
    "Locrian",
    "Blues",
    "Hollywood",
    "Chromatic",
]

def setScaleToKeyword(trackId, keyword):
    """Use a builtin base scale and apply to all notes in a pattern. If there are more more ore
    fewer notes in the pattern than in the scale we will calculate the rest.

    This function is called not often and does not need to be performant.
    """
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.scale[:]: setScale(trId, v, callback=True), descriptionString="Set Scale")

    rememberRootNote = track.pattern.scale[-1] #The last note has a special role by convention. No matter if this is the lowest midi-pitch or not. Most of the time it is the lowest though.

    #Create a modified scalePattern for the tracks numberOfSteps.
    #We technically only need to worry about creating additional steps. less steps is covered by zip(), see below
    majorExt = []
    schemeExt = []
    mrev = list(reversed(major*16)) #pad major to the maximum possible notes. We just need the basis to make it possible long schemes like chromatic fit
    srev = list(reversed(schemesDict[keyword]*16))
    for i in range(track.pattern.numberOfSteps):
        l = len(srev)
        octaveOffset = i // l * 12 #starts with 0*12
        majorExt.append( mrev[i % l ] + octaveOffset)
        schemeExt.append( srev[i % l] )  #this is always the same. it is only the difference to the major scale

    majorExt = list(reversed(majorExt))
    schemeExt = list(reversed(schemeExt))

    scale = [x + y for x, y in zip(majorExt, schemeExt)] #zip just creates pairs until it reached the end of one of its arguments. This is reversed order.
    #scale = [x + y for x, y in zip(major, schemesDict[keyword])] #zip just creates pairs until it reached the end of one of its arguments. This is reversed order.
    difference = rememberRootNote - scale[-1] #isn't this the same as rootnote since scale[-1] is always 0? Well, we could have hypo-scales in the future.
    result = [midipitch+difference for midipitch in scale] #create actual midi pitches from the root note and the scale.  This is reversed order because "scale" is.

    #Here is a hack because chromatic didn't work with octave wrap-around. We want to make sure we don't fall back to a lower octave
    r = reversed(result)
    result = []
    oldPitch = 0
    for p in r:
        while p < oldPitch:
            p += 12
        result.append(p)
        oldPitch = p
    result = reversed(result)

    #Done. Inform all parties.
    track.pattern.scale = result
    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._trackMetaDataChanged(track)

def changePatternVelocity(trackId, steps):
    track = session.data.trackById(trackId)
    if not track: return
    session.history.register(lambda trId=trackId, v=track.pattern.copyData(): setPattern(trId, v, "Change Pattern Velocity"), descriptionString="Change Pattern Velocity")
    for note in track.pattern.data:
        new = note["velocity"] + steps
        note["velocity"] = min(max(new,0), 127)

    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)

def resizePatternWithoutScale(trackId, steps):
    """Resize a patterns number of steps without changing the scale.
    Can't go below 1 step.

    Our editing end is the bottom one, where new steps are removed or added.

    We also cannot know if there the user set a scale through an api scheme. At the very least
    this needs analyzing and taking an educated guess. For now we just add notes a semitone below.
    """
    if steps < 1 or steps > 128:
        logger.warning(f"Pattern must have >= 1 and <= 127 steps but {steps} was requested. Doing nothing.")
        return

    track = session.data.trackById(trackId)
    if not track: return

    #We could use setScale for undo. But this requires a different set of callbacks. We use our own function, eventhough some of the calculations are not needed for undo.

    session.history.register(lambda trId=trackId, v=track.pattern.numberOfSteps: resizePatternWithoutScale(trId, v), descriptionString="Number of Notes in Pattern")

    currentNr = track.pattern.numberOfSteps #int
    oldid = id(track.pattern.scale)
    s = track.pattern.scale #GUI view: from top to bottom. Usually from higher pitches to lower. (49, 53, 50, 45, 42, 39, 38, 36)
    if steps == currentNr:
        return

    if steps < currentNr: #just reduce
        track.pattern.scale = tuple(s[:steps])
    else: #new
        currentLowest = s[-1] #int
        result = list(s) #can be edited.
        for i in range(steps-currentNr):
            currentLowest -= 1
            result.append(currentLowest)
        track.pattern.scale = tuple(result)
        assert track.pattern.numberOfSteps == steps, (track.pattern.numberOfSteps, steps)

    assert not oldid == id(track.pattern.scale)

    track.pattern.buildExportCache()
    track.buildTrack()
    updatePlayback()
    callbacks._patternChanged(track)


#Other functions. These can't be template functions because they use a specific track and Patroneos row and scale system.
def noteOn(trackId, row):
    track = session.data.trackById(trackId)
    if not track: return
    try:
        midipitch = track.pattern.scale[row]
        cbox.send_midi_event(0x90+track.midiChannel, midipitch, track.pattern.averageVelocity, output=track.cboxMidiOutAbstraction)
    except IndexError: #We got a note that is not in the current scale and pattern. E.g. because numberOfSteps is 7 but we received 8.
        pass

def noteOff(trackId, row):
    track = session.data.trackById(trackId)
    if not track: return
    try:
        midipitch = track.pattern.scale[row]
        cbox.send_midi_event(0x80+track.midiChannel, midipitch, track.pattern.averageVelocity, output=track.cboxMidiOutAbstraction)
    except IndexError: #We got a note that is not in the current scale and pattern. E.g. because numberOfSteps is 7 but we received 8.
        pass