Overhaul programfinder. More reliable, faster, gives feedback

engine/api.py

@@ -197,10 +197,10 @@ def sessionList()->list:
     r = nsmServerControl.exportSessionsAsDicts()
     return [s["nsmSessionName"] for s in r]
 
-def buildSystemPrograms():
+def buildSystemPrograms(progressHook=None):
     """Build a list of dicts with the .desktop files (or similar) of all NSM compatible programs
     present on the system"""
-    programDatabase.build()
+    programDatabase.build(progressHook)
 
 def systemProgramsSetWhitelist(executableNames:tuple):
     """will replace the current list"""

engine/findprograms.py

@@ -29,9 +29,15 @@ import stat
 
 from engine.start import PATHS
 
+def nothing(*args): pass
+
 class SupportedProgramsDatabase(object):
-    """Find all binaries. Use all available resources: xdg desktop files, binary string seach in
-    executables, data bases, supplement with user choices.
+    """Find all binaries with NSM support. Resources are:
+    * Argodejo internal program list of known working programs.
+    * Internal blacklist of known redundant programs (such as non-daw) or nonsense entries, like Argodejo itself    
+    * A search through the users path to find stray programs that contain NSM announce messages
+    * Finally, local to a users system: User whitelist for any program, user blacklist.
+    Those two have the highest priority.
 
     We generate the same format as configParser does with .desktop files as _sections dict.
 
@@ -50,16 +56,25 @@ class SupportedProgramsDatabase(object):
 
     In case there is a file in PATH or database but has no .desktop we create our own entry with
     missing data.
+
     """
 
     def __init__(self):        
+        self.progressHook = nothing #prevents the initial programstart from sending meaningless messages for the cached data. Set and reverted in self.build
         
-        #self.grepexcluded = (pathlib.Path(PATHS["share"], "grepexcluded.txt")) #from a system without audio software installed:  find /usr/bin  -printf "%f\n" | sort > grepexcluded.txt 
+        self.grepexcluded = (pathlib.Path(PATHS["share"], "grepexcluded.txt")) #created by hand. see docstring
         #assert self.grepexcluded.exists()
-        self.blacklist = ("nsmd", "non-daw", "carla") #only programs that have to do with audio and music. There is another general blacklist that speeds up discovery
-        self.morePrograms = ("thisdoesnotexisttest", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour5", "ardour6", "nsm-data", "nsm-jack") #only programs not found by buildCache_grepExecutablePaths because they are just shellscripts and do not contain /nsm/server/announce.
-        self.userWhitelist = () #added dynamically to morePrograms
-        self.userBlacklist = () #added dynamically to blacklist
+        self.blacklist = ("nsmd", "non-daw", "carla", "argodejo", "adljack") #only programs that have to do with audio and music. There is another general blacklist that speeds up discovery
+        self.whiteList = ("thisdoesnotexisttest", "patroneo", "vico",
+                    "fluajho", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour5",
+                    "ardour6", "nsm-data", "jackpatch", "nsm-proxy", "ADLplug", "ams",
+                    "drumkv1_jack", "synthv1_jack", "samplv1_jack", "padthv1_jack",
+                    "luppp", "non-mixer", "non-timeline", "non-sequencer", "non-midi-mapper", "non-mixer-noui",
+                    "OPNplug", "qmidiarp", "qtractor", "zynaddsubfx", "jack_mixer",
+                    "hydrogen", "mfp", "shuriken", "laborejo", "guitarix", "radium"                    
+                    ) #shortcut list and programs not found by buildCache_grepExecutablePaths because they are just shellscripts and do not contain /nsm/server/announce.
+        self.userWhitelist = () #added dynamically to morePrograms. highest priority
+        self.userBlacklist = () #added dynamically to blacklist. highest priority
         self.knownDesktopFiles = { #shortcuts to the correct desktop files. Reverse lookup binary->desktop creates false entries, for example ZynAddSubFx and Carla.
             "zynaddsubfx": "zynaddsubfx-jack.desktop", #value will later get replaced with the .desktop entry
             "carla-jack-multi" : "carla.desktop",
@@ -71,8 +86,8 @@ class SupportedProgramsDatabase(object):
         self.unfilteredExecutables = None #in build()
         #self.build() #fills self.programs and
 
-    def buildCache_grepExecutablePaths(self): #TODO: this is a time consuming process. But is has a chance of finding programs that would have been missed otherwise.
-        """return a list of executable names in the path
+    def buildCache_grepExecutablePaths(self)->list:
+        """return a list of executable names in the path (not the path itself)
         Grep explained:
         -s silent. No errors, eventhough subprocess uses stdout only
         -R recursive with symlinks. We don't want to look in subdirs because that is not allowed by
@@ -83,25 +98,34 @@ class SupportedProgramsDatabase(object):
         Your binaries will be in unfilteredExecutables though
         """
         result = []
-        executablePaths = [pathlib.Path(p) for p in os.environ["PATH"].split(os.pathsep)]
+        testpaths = os.environ["PATH"].split(os.pathsep) + ["/bin", "/sbin"]
+        executablePaths = set([pathlib.Path(p).resolve() for p in os.environ["PATH"].split(os.pathsep)]) #resolve filters out symlinks, like arches /sbin and /bin. set() makes it unique
+
+        excludeFromProcessingSet = set(self.blacklist + self.userBlacklist)
+        whiteSet = set(self.whiteList + self.userWhitelist)
+        excludeFromProcessingSet.update(whiteSet)
+        
         for path in executablePaths:
-            #command = f"grep --exclude-from {self.grepexcluded}  -iRsnl {path} -e /nsm/server/announce"           
-            command = f"grep -iRsnl {path} -e /nsm/server/announce"           
+            self.progressHook(f"{path}")
+            command = f"grep --exclude-from {self.grepexcluded}  -iRsnl {path} -e /nsm/server/announce"           
+            #command = f"grep -iRsnl {path} -e /nsm/server/announce"           
                         
             #Py>=3.7 completedProcess = subprocess.run(command, capture_output=True, text=True, shell=True)
             completedProcess = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) #universal_newlines is an alias for text, which was deprecated in 3.7 because text is more understandable.  capture_output replaces the two PIPEs in 3.7
             for fullPath in completedProcess.stdout.split():
+                self.progressHook(f"{fullPath}")
                 exe = pathlib.Path(fullPath).relative_to(path)               
-                if not str(exe) in self.blacklist + self.userBlacklist:
+                if not str(exe) in excludeFromProcessingSet: #skip over any known file, good or bad
                     result.append((str(exe), str(fullPath)))
 
-        for prg in self.morePrograms + self.userWhitelist:
+        for prg in whiteSet:
+            self.progressHook(f"{prg}")
             for path in executablePaths:
-                if pathlib.Path(path, prg).is_file():
+                if pathlib.Path(path, prg).is_file(): #check if this actually exists
                     result.append((str(prg), str(pathlib.Path(path, prg))))
                     break #inner loop
         
-        return result
+        return list(set(result)) #make unique
 
     def buildCache_DesktopEntries(self):
         """Go through all dirs including subdirs"""
@@ -115,13 +139,11 @@ class SupportedProgramsDatabase(object):
         allDesktopEntries = []
         for basePath in xdgPaths:
             for f in basePath.glob('**/*'):
+                self.progressHook(f"{f}")
                 if f.is_file() and f.suffix == ".desktop":
                     config.clear()
                     try:
                         config.read(f)
-                    except: #any bad config means skip
-                        continue #next file
-                    try:
                         entryDict = dict(config._sections["Desktop Entry"])
                         if f.name == "zynaddsubfx-jack.desktop":
                             self.knownDesktopFiles["zynaddsubfx"] = entryDict
@@ -129,7 +151,7 @@ class SupportedProgramsDatabase(object):
                             self.knownDesktopFiles["carla-jack-multi"] = entryDict
                         #in any case:
                         allDesktopEntries.append(entryDict)  #_sections   'DesktopEntry':{dictOfActualData)
-                    except:
+                    except: #any bad config means skip
                         logger.warning(f"Bad desktop file. Skipping: {f}")                        
         return allDesktopEntries
 
@@ -138,13 +160,24 @@ class SupportedProgramsDatabase(object):
         self.programs = listOfDicts
         self.nsmExecutables = set(d["argodejoExec"] for d in self.programs)
 
-    def build(self):
+    def build(self, progressHook=None):
         """Can be called at any time by the user to update after installing new programs"""
+
+        if progressHook:
+            #receives one string which indicates what files is currently parsed.
+            #Just the pure path, this will not get translated!
+            #The purpose is to show a "we are not frozen!" feedback to the user.
+            #It doesn't really matter what is reported back as long as it changes often            
+            self.progressHook = progressHook 
+
         logger.info("Building launcher database. This might take a minute")
+        self.progressHook("")
         self.programs = self._build()
         self.unfilteredExecutables = self.buildCache_unfilteredExecutables()
         self.nsmExecutables = set(d["argodejoExec"] for d in self.programs)
         self._buildWhitelist()
+        self.progressHook("")
+        self.progressHook = nothing
         logger.info("Building launcher database done.")
 
     def _exeToDesktopEntry(self, exe:str)->dict:
@@ -168,6 +201,7 @@ class SupportedProgramsDatabase(object):
         leftovers = set(self.executables)
         matches = [] #list of dicts
         for exe, fullPath in self.executables:
+            self.progressHook(f"{fullPath}")
             entry = self._exeToDesktopEntry(exe)
             if entry: #Found match!
                 entry["argodejoFullPath"] = fullPath
@@ -212,10 +246,10 @@ class SupportedProgramsDatabase(object):
         def isexe(path):
             """executable by owner"""
             return path.is_file() and stat.S_IXUSR & os.stat(path)[stat.ST_MODE] == 64
-
         result = []
         executablePaths = [pathlib.Path(p) for p in os.environ["PATH"].split(os.pathsep)]
         for path in executablePaths:
+            self.progressHook(f"{path}")            
             result += [str(pathlib.Path(f).relative_to(path)) for f in path.glob("*") if isexe(f)]
 
         return sorted(list(set(result)))
@@ -225,11 +259,10 @@ class SupportedProgramsDatabase(object):
         It will be populated from a template-list of well-working clients and then all binaries not
         in the path are filtered out. This can be presented to the user without worries."""
         #Assumes to be called only from self.build
-        startexecutables = set(("doesnotexist", "laborejo", "patroneo", "vico", "fluajho", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour6", "drumkv1_jack", "synthv1_jack", "padthv1_jack", "samplv1_jack", "zynaddsubfx", "ADLplug", "OPNplug", "non-mixer", "non-sequencer", "non-timeline") + self.userWhitelist)
+        startexecutables = set(self.whiteList + self.userWhitelist)
         for prog in self.programs:
             prog["whitelist"] = prog["argodejoExec"] in startexecutables
 
-
         """
         matches = []
         for exe in startexecutables:

engine/resources/gen_grepexcluded.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+sudo pacman -Fy
+pacman -Fx "/usr/bin/([A-Z])" | cut -d " " -f1 | uniq | sort > allexe.txt
+pacman -Fl $(pacman -Sg pro-audio |cut -d " " -f2) | cut -d " " -f2 | grep "usr/bin" | uniq | sort > audioexe.txt
+sed -i -e 's/usr\/bin\///g' audioexe.txt   #strip usr/bin  yes, the initial / from usr is missing
+sed '/^[[:space:]]*$/d' -i audioexe.txt #remove empty lines
+
+grep -vFf audioexe.txt allexe.txt > grepexcluded2.txt
+grep '^usr\/bin\/' grepexcluded2.txt > grepexcluded.txt #only keep lines that start with usr/bin. There are some false positives in there
+
+sed -i -e 's/usr\/bin\///g' grepexcluded.txt   #strip usr/bin  yes, the initial / from usr is missing
+sed '/^[[:space:]]*$/d' -i grepexcluded.txt #remove empty lines
+
+rm grepexcluded2.txt
+rm allexe.txt
+rm audioexe.txt

qtgui/designer/mainwindow.py

@@ -392,7 +392,7 @@ class Ui_MainWindow(object):
         self.actionSessionSaveAs.setShortcut(_translate("MainWindow", "Ctrl+Shift+S"))
         self.actionSessionSaveAndClose.setText(_translate("MainWindow", "Save and Close"))
         self.actionSessionSaveAndClose.setShortcut(_translate("MainWindow", "Ctrl+W"))
-        self.actionSessionAbort.setText(_translate("MainWindow", "Abort"))
+        self.actionSessionAbort.setText(_translate("MainWindow", "Close without Save (\"Abort\")"))
         self.actionSessionAbort.setShortcut(_translate("MainWindow", "Ctrl+Shift+W"))
         self.actionClientStop.setText(_translate("MainWindow", "Stop"))
         self.actionClientStop.setShortcut(_translate("MainWindow", "Alt+O"))

qtgui/designer/mainwindow.ui

@@ -728,7 +728,7 @@
   </action>
   <action name="actionSessionAbort">
    <property name="text">
-    <string>Abort</string>
+    <string>Close without Save (&quot;Abort&quot;)</string>
    </property>
    <property name="shortcut">
     <string>Ctrl+Shift+W</string>

qtgui/resources/translations/de.ts

@@ -4,32 +4,32 @@
 <context>
     <name>AskBeforeQuit</name>
     <message>
-        <location filename="../../mainwindow.py" line="256"/>
+        <location filename="../../mainwindow.py" line="290"/>
         <source>About to quit but session {} still open</source>
         <translation>Programm soll beendet werden, aber Session {} ist noch offen</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="257"/>
+        <location filename="../../mainwindow.py" line="291"/>
         <source>Do you want to save?</source>
         <translation>Möchten Sie speichern?</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="258"/>
+        <location filename="../../mainwindow.py" line="292"/>
         <source>About to quit</source>
         <translation>Programm soll beendet werden</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="267"/>
+        <location filename="../../mainwindow.py" line="301"/>
         <source>Don&apos;t Quit</source>
         <translation>Nicht Beenden</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="268"/>
+        <location filename="../../mainwindow.py" line="302"/>
         <source>Save</source>
         <translation>Speichern</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="269"/>
+        <location filename="../../mainwindow.py" line="303"/>
         <source>Discard Changes</source>
         <translation>Änderungen Verwerfen</translation>
     </message>
@@ -305,7 +305,7 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
     <message>
         <location filename="../../designer/mainwindow.py" line="395"/>
         <source>Abort</source>
-        <translation>Abbrechen</translation>
+        <translation type="obsolete">Abbrechen</translation>
     </message>
     <message>
         <location filename="../../designer/mainwindow.py" line="396"/>
@@ -397,6 +397,11 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
         <source>Save and Clone under different name</source>
         <translation>Speichern und mit anderem Namen neu öffnen</translation>
     </message>
+    <message>
+        <location filename="../../designer/mainwindow.py" line="395"/>
+        <source>Close without Save (&quot;Abort&quot;)</source>
+        <translation>Schließen ohne zu Speichern (&quot;Abbrechen&quot;)</translation>
+    </message>
 </context>
 <context>
     <name>NewSession</name>
@@ -478,13 +483,11 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
     <message>
         <location filename="../../opensessioncontroller.py" line="86"/>
         <source>✔</source>
-        <translatorcomment>✔</translatorcomment>
         <translation>✔</translation>
     </message>
     <message>
         <location filename="../../opensessioncontroller.py" line="88"/>
         <source>✖</source>
-        <translatorcomment>✖</translatorcomment>
         <translation>✖</translation>
     </message>
 </context>
@@ -555,72 +558,72 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
 <context>
     <name>SessionTree</name>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="144"/>
+        <location filename="../../sessiontreecontroller.py" line="145"/>
         <source>Name</source>
         <translation>Name</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="145"/>
+        <location filename="../../sessiontreecontroller.py" line="146"/>
         <source>Last Save</source>
         <translation>Letzte Speicherung</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="146"/>
+        <location filename="../../sessiontreecontroller.py" line="147"/>
         <source>Clients</source>
         <translation>Clients</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="147"/>
+        <location filename="../../sessiontreecontroller.py" line="148"/>
         <source>Size</source>
         <translation>Größe</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="148"/>
+        <location filename="../../sessiontreecontroller.py" line="149"/>
         <source>Symlinks</source>
         <translation>Symlinks</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="149"/>
+        <location filename="../../sessiontreecontroller.py" line="150"/>
         <source>Path</source>
         <translation>Pfad</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="246"/>
+        <location filename="../../sessiontreecontroller.py" line="247"/>
         <source>About to delete Session {}</source>
         <translation>Session {} soll gelöscht werden</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="245"/>
+        <location filename="../../sessiontreecontroller.py" line="246"/>
         <source>All files in the project directory will be irreversibly deleted.</source>
         <translation>Alle Dateien aus diesem Projektverzeichnis werden unwiederbringlich gelöscht.</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="254"/>
+        <location filename="../../sessiontreecontroller.py" line="255"/>
         <source>Keep Session</source>
         <translation>Session behalten</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="255"/>
+        <location filename="../../sessiontreecontroller.py" line="256"/>
         <source>Delete!</source>
         <translation>Löschen!</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="271"/>
+        <location filename="../../sessiontreecontroller.py" line="272"/>
         <source>Copy Session</source>
         <translation>Session kopieren</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="274"/>
+        <location filename="../../sessiontreecontroller.py" line="275"/>
         <source>Force Lock Removal</source>
         <translation>Lockdatei Aufhebung erzwingen</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="276"/>
+        <location filename="../../sessiontreecontroller.py" line="277"/>
         <source>Rename Session</source>
         <translation>Session umbennen</translation>
     </message>
     <message>
-        <location filename="../../sessiontreecontroller.py" line="278"/>
+        <location filename="../../sessiontreecontroller.py" line="279"/>
         <source>Delete Session</source>
         <translation>Session löschen</translation>
     </message>
@@ -628,22 +631,22 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
 <context>
     <name>TrayIcon</name>
     <message>
-        <location filename="../../systemtray.py" line="68"/>
+        <location filename="../../systemtray.py" line="64"/>
         <source>Hide/Show Argodejo</source>
         <translation>Verstecke/Zeige Argodejo</translation>
     </message>
     <message>
-        <location filename="../../systemtray.py" line="81"/>
+        <location filename="../../systemtray.py" line="77"/>
         <source>Save &amp;&amp; Quit Argodejo</source>
         <translation>Speichern und Argodejo Beenden</translation>
     </message>
     <message>
-        <location filename="../../systemtray.py" line="82"/>
+        <location filename="../../systemtray.py" line="78"/>
         <source>Abort &amp;&amp; Quit Argodejo</source>
         <translation>Abbrechen und Argodejo Beenden</translation>
     </message>
     <message>
-        <location filename="../../systemtray.py" line="88"/>
+        <location filename="../../systemtray.py" line="84"/>
         <source>Quit </source>
         <translation>Beenden </translation>
     </message>
@@ -651,17 +654,17 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
 <context>
     <name>mainWindow</name>
     <message>
-        <location filename="../../mainwindow.py" line="157"/>
+        <location filename="../../mainwindow.py" line="188"/>
         <source>Argodejo ready</source>
         <translation>Argodejo ist bereit</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="192"/>
+        <location filename="../../mainwindow.py" line="223"/>
         <source>Another GUI tried to launch.</source>
         <translation>Es wurde versucht eine weitere GUI zu starten.</translation>
     </message>
     <message>
-        <location filename="../../mainwindow.py" line="219"/>
+        <location filename="../../mainwindow.py" line="251"/>
         <source>Updating Program Database.
 Thank you for your patience.</source>
         <translation>Programmdatenbank wird aktualisiert.

qtgui/waitdialog.py

@@ -54,13 +54,17 @@ class WaitDialog(object):
     def __init__(self, mainWindow, text, longRunningFunction):
 
         super().__init__()
+        self.text = text
         logger.info(f"Starting blocking message for {longRunningFunction}")
         self.mainWindow = mainWindow
         self.mainWindow.ui.messageLabel.setText(text)
         self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(1) #1 is messageLabel 0 is the tab widget
         self.mainWindow.ui.menubar.setEnabled(False) #TODO: this will leave the options in the TrayIcon menu available.. but well, who cares...
 
-        wt = WaitThread(mainWindow, longRunningFunction)
+        def wrap():
+            longRunningFunction(self.progressInfo)
+       
+        wt = WaitThread(mainWindow, wrap)
         #wt.finished.connect(self.threadDone) #does NOT trigger
         wt.start()
         while not wt.finished:
@@ -70,3 +74,6 @@ class WaitDialog(object):
         self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(0) #1 is messageLabel 0 is the tab widget      
 
 
+    def progressInfo(self, path:str):
+        self.mainWindow.ui.messageLabel.setText(self.text + "\n\n" + path) 
+