Browse Source

Overhaul programfinder. More reliable, faster, gives feedback

master
Nils 3 years ago
parent
commit
19a204d9ca
  1. 4
      engine/api.py
  2. 89
      engine/findprograms.py
  3. 17
      engine/resources/gen_grepexcluded.sh
  4. 17604
      engine/resources/grepexcluded.txt
  5. 2
      qtgui/designer/mainwindow.py
  6. 2
      qtgui/designer/mainwindow.ui
  7. 1366
      qtgui/resources.py
  8. BIN
      qtgui/resources/translations/de.qm
  9. 63
      qtgui/resources/translations/de.ts
  10. 11
      qtgui/waitdialog.py

4
engine/api.py

@ -197,10 +197,10 @@ def sessionList()->list:
r = nsmServerControl.exportSessionsAsDicts()
return [s["nsmSessionName"] for s in r]
def buildSystemPrograms():
def buildSystemPrograms(progressHook=None):
"""Build a list of dicts with the .desktop files (or similar) of all NSM compatible programs
present on the system"""
programDatabase.build()
programDatabase.build(progressHook)
def systemProgramsSetWhitelist(executableNames:tuple):
"""will replace the current list"""

89
engine/findprograms.py

@ -29,9 +29,15 @@ import stat
from engine.start import PATHS
def nothing(*args): pass
class SupportedProgramsDatabase(object):
"""Find all binaries. Use all available resources: xdg desktop files, binary string seach in
executables, data bases, supplement with user choices.
"""Find all binaries with NSM support. Resources are:
* Argodejo internal program list of known working programs.
* Internal blacklist of known redundant programs (such as non-daw) or nonsense entries, like Argodejo itself
* A search through the users path to find stray programs that contain NSM announce messages
* Finally, local to a users system: User whitelist for any program, user blacklist.
Those two have the highest priority.
We generate the same format as configParser does with .desktop files as _sections dict.
@ -50,16 +56,25 @@ class SupportedProgramsDatabase(object):
In case there is a file in PATH or database but has no .desktop we create our own entry with
missing data.
"""
def __init__(self):
"""
#self.grepexcluded = (pathlib.Path(PATHS["share"], "grepexcluded.txt")) #from a system without audio software installed: find /usr/bin -printf "%f\n" | sort > grepexcluded.txt
def __init__(self):
self.progressHook = nothing #prevents the initial programstart from sending meaningless messages for the cached data. Set and reverted in self.build
self.grepexcluded = (pathlib.Path(PATHS["share"], "grepexcluded.txt")) #created by hand. see docstring
#assert self.grepexcluded.exists()
self.blacklist = ("nsmd", "non-daw", "carla") #only programs that have to do with audio and music. There is another general blacklist that speeds up discovery
self.morePrograms = ("thisdoesnotexisttest", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour5", "ardour6", "nsm-data", "nsm-jack") #only programs not found by buildCache_grepExecutablePaths because they are just shellscripts and do not contain /nsm/server/announce.
self.userWhitelist = () #added dynamically to morePrograms
self.userBlacklist = () #added dynamically to blacklist
self.blacklist = ("nsmd", "non-daw", "carla", "argodejo", "adljack") #only programs that have to do with audio and music. There is another general blacklist that speeds up discovery
self.whiteList = ("thisdoesnotexisttest", "patroneo", "vico",
"fluajho", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour5",
"ardour6", "nsm-data", "jackpatch", "nsm-proxy", "ADLplug", "ams",
"drumkv1_jack", "synthv1_jack", "samplv1_jack", "padthv1_jack",
"luppp", "non-mixer", "non-timeline", "non-sequencer", "non-midi-mapper", "non-mixer-noui",
"OPNplug", "qmidiarp", "qtractor", "zynaddsubfx", "jack_mixer",
"hydrogen", "mfp", "shuriken", "laborejo", "guitarix", "radium"
) #shortcut list and programs not found by buildCache_grepExecutablePaths because they are just shellscripts and do not contain /nsm/server/announce.
self.userWhitelist = () #added dynamically to morePrograms. highest priority
self.userBlacklist = () #added dynamically to blacklist. highest priority
self.knownDesktopFiles = { #shortcuts to the correct desktop files. Reverse lookup binary->desktop creates false entries, for example ZynAddSubFx and Carla.
"zynaddsubfx": "zynaddsubfx-jack.desktop", #value will later get replaced with the .desktop entry
"carla-jack-multi" : "carla.desktop",
@ -71,8 +86,8 @@ class SupportedProgramsDatabase(object):
self.unfilteredExecutables = None #in build()
#self.build() #fills self.programs and
def buildCache_grepExecutablePaths(self): #TODO: this is a time consuming process. But is has a chance of finding programs that would have been missed otherwise.
"""return a list of executable names in the path
def buildCache_grepExecutablePaths(self)->list:
"""return a list of executable names in the path (not the path itself)
Grep explained:
-s silent. No errors, eventhough subprocess uses stdout only
-R recursive with symlinks. We don't want to look in subdirs because that is not allowed by
@ -83,25 +98,34 @@ class SupportedProgramsDatabase(object):
Your binaries will be in unfilteredExecutables though
"""
result = []
executablePaths = [pathlib.Path(p) for p in os.environ["PATH"].split(os.pathsep)]
testpaths = os.environ["PATH"].split(os.pathsep) + ["/bin", "/sbin"]
executablePaths = set([pathlib.Path(p).resolve() for p in os.environ["PATH"].split(os.pathsep)]) #resolve filters out symlinks, like arches /sbin and /bin. set() makes it unique
excludeFromProcessingSet = set(self.blacklist + self.userBlacklist)
whiteSet = set(self.whiteList + self.userWhitelist)
excludeFromProcessingSet.update(whiteSet)
for path in executablePaths:
#command = f"grep --exclude-from {self.grepexcluded} -iRsnl {path} -e /nsm/server/announce"
command = f"grep -iRsnl {path} -e /nsm/server/announce"
self.progressHook(f"{path}")
command = f"grep --exclude-from {self.grepexcluded} -iRsnl {path} -e /nsm/server/announce"
#command = f"grep -iRsnl {path} -e /nsm/server/announce"
#Py>=3.7 completedProcess = subprocess.run(command, capture_output=True, text=True, shell=True)
completedProcess = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) #universal_newlines is an alias for text, which was deprecated in 3.7 because text is more understandable. capture_output replaces the two PIPEs in 3.7
for fullPath in completedProcess.stdout.split():
exe = pathlib.Path(fullPath).relative_to(path)
if not str(exe) in self.blacklist + self.userBlacklist:
self.progressHook(f"{fullPath}")
exe = pathlib.Path(fullPath).relative_to(path)
if not str(exe) in excludeFromProcessingSet: #skip over any known file, good or bad
result.append((str(exe), str(fullPath)))
for prg in self.morePrograms + self.userWhitelist:
for prg in whiteSet:
self.progressHook(f"{prg}")
for path in executablePaths:
if pathlib.Path(path, prg).is_file():
if pathlib.Path(path, prg).is_file(): #check if this actually exists
result.append((str(prg), str(pathlib.Path(path, prg))))
break #inner loop
return result
return list(set(result)) #make unique
def buildCache_DesktopEntries(self):
"""Go through all dirs including subdirs"""
@ -115,13 +139,11 @@ class SupportedProgramsDatabase(object):
allDesktopEntries = []
for basePath in xdgPaths:
for f in basePath.glob('**/*'):
self.progressHook(f"{f}")
if f.is_file() and f.suffix == ".desktop":
config.clear()
try:
config.read(f)
except: #any bad config means skip
continue #next file
try:
entryDict = dict(config._sections["Desktop Entry"])
if f.name == "zynaddsubfx-jack.desktop":
self.knownDesktopFiles["zynaddsubfx"] = entryDict
@ -129,8 +151,8 @@ class SupportedProgramsDatabase(object):
self.knownDesktopFiles["carla-jack-multi"] = entryDict
#in any case:
allDesktopEntries.append(entryDict) #_sections 'DesktopEntry':{dictOfActualData)
except:
logger.warning(f"Bad desktop file. Skipping: {f}")
except: #any bad config means skip
logger.warning(f"Bad desktop file. Skipping: {f}")
return allDesktopEntries
def loadPrograms(self, listOfDicts):
@ -138,13 +160,24 @@ class SupportedProgramsDatabase(object):
self.programs = listOfDicts
self.nsmExecutables = set(d["argodejoExec"] for d in self.programs)
def build(self):
def build(self, progressHook=None):
"""Can be called at any time by the user to update after installing new programs"""
if progressHook:
#receives one string which indicates what files is currently parsed.
#Just the pure path, this will not get translated!
#The purpose is to show a "we are not frozen!" feedback to the user.
#It doesn't really matter what is reported back as long as it changes often
self.progressHook = progressHook
logger.info("Building launcher database. This might take a minute")
self.progressHook("")
self.programs = self._build()
self.unfilteredExecutables = self.buildCache_unfilteredExecutables()
self.nsmExecutables = set(d["argodejoExec"] for d in self.programs)
self._buildWhitelist()
self.progressHook("")
self.progressHook = nothing
logger.info("Building launcher database done.")
def _exeToDesktopEntry(self, exe:str)->dict:
@ -168,6 +201,7 @@ class SupportedProgramsDatabase(object):
leftovers = set(self.executables)
matches = [] #list of dicts
for exe, fullPath in self.executables:
self.progressHook(f"{fullPath}")
entry = self._exeToDesktopEntry(exe)
if entry: #Found match!
entry["argodejoFullPath"] = fullPath
@ -212,10 +246,10 @@ class SupportedProgramsDatabase(object):
def isexe(path):
"""executable by owner"""
return path.is_file() and stat.S_IXUSR & os.stat(path)[stat.ST_MODE] == 64
result = []
executablePaths = [pathlib.Path(p) for p in os.environ["PATH"].split(os.pathsep)]
for path in executablePaths:
self.progressHook(f"{path}")
result += [str(pathlib.Path(f).relative_to(path)) for f in path.glob("*") if isexe(f)]
return sorted(list(set(result)))
@ -225,11 +259,10 @@ class SupportedProgramsDatabase(object):
It will be populated from a template-list of well-working clients and then all binaries not
in the path are filtered out. This can be presented to the user without worries."""
#Assumes to be called only from self.build
startexecutables = set(("doesnotexist", "laborejo", "patroneo", "vico", "fluajho", "carla-rack", "carla-patchbay", "carla-jack-multi", "ardour6", "drumkv1_jack", "synthv1_jack", "padthv1_jack", "samplv1_jack", "zynaddsubfx", "ADLplug", "OPNplug", "non-mixer", "non-sequencer", "non-timeline") + self.userWhitelist)
startexecutables = set(self.whiteList + self.userWhitelist)
for prog in self.programs:
prog["whitelist"] = prog["argodejoExec"] in startexecutables
"""
matches = []
for exe in startexecutables:

17
engine/resources/gen_grepexcluded.sh

@ -0,0 +1,17 @@
#!/bin/sh
sudo pacman -Fy
pacman -Fx "/usr/bin/([A-Z])" | cut -d " " -f1 | uniq | sort > allexe.txt
pacman -Fl $(pacman -Sg pro-audio |cut -d " " -f2) | cut -d " " -f2 | grep "usr/bin" | uniq | sort > audioexe.txt
sed -i -e 's/usr\/bin\///g' audioexe.txt #strip usr/bin yes, the initial / from usr is missing
sed '/^[[:space:]]*$/d' -i audioexe.txt #remove empty lines
grep -vFf audioexe.txt allexe.txt > grepexcluded2.txt
grep '^usr\/bin\/' grepexcluded2.txt > grepexcluded.txt #only keep lines that start with usr/bin. There are some false positives in there
sed -i -e 's/usr\/bin\///g' grepexcluded.txt #strip usr/bin yes, the initial / from usr is missing
sed '/^[[:space:]]*$/d' -i grepexcluded.txt #remove empty lines
rm grepexcluded2.txt
rm allexe.txt
rm audioexe.txt

17604
engine/resources/grepexcluded.txt

File diff suppressed because it is too large

2
qtgui/designer/mainwindow.py

@ -392,7 +392,7 @@ class Ui_MainWindow(object):
self.actionSessionSaveAs.setShortcut(_translate("MainWindow", "Ctrl+Shift+S"))
self.actionSessionSaveAndClose.setText(_translate("MainWindow", "Save and Close"))
self.actionSessionSaveAndClose.setShortcut(_translate("MainWindow", "Ctrl+W"))
self.actionSessionAbort.setText(_translate("MainWindow", "Abort"))
self.actionSessionAbort.setText(_translate("MainWindow", "Close without Save (\"Abort\")"))
self.actionSessionAbort.setShortcut(_translate("MainWindow", "Ctrl+Shift+W"))
self.actionClientStop.setText(_translate("MainWindow", "Stop"))
self.actionClientStop.setShortcut(_translate("MainWindow", "Alt+O"))

2
qtgui/designer/mainwindow.ui

@ -728,7 +728,7 @@
</action>
<action name="actionSessionAbort">
<property name="text">
<string>Abort</string>
<string>Close without Save (&quot;Abort&quot;)</string>
</property>
<property name="shortcut">
<string>Ctrl+Shift+W</string>

1366
qtgui/resources.py

File diff suppressed because it is too large

BIN
qtgui/resources/translations/de.qm

Binary file not shown.

63
qtgui/resources/translations/de.ts

@ -4,32 +4,32 @@
<context>
<name>AskBeforeQuit</name>
<message>
<location filename="../../mainwindow.py" line="256"/>
<location filename="../../mainwindow.py" line="290"/>
<source>About to quit but session {} still open</source>
<translation>Programm soll beendet werden, aber Session {} ist noch offen</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="257"/>
<location filename="../../mainwindow.py" line="291"/>
<source>Do you want to save?</source>
<translation>Möchten Sie speichern?</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="258"/>
<location filename="../../mainwindow.py" line="292"/>
<source>About to quit</source>
<translation>Programm soll beendet werden</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="267"/>
<location filename="../../mainwindow.py" line="301"/>
<source>Don&apos;t Quit</source>
<translation>Nicht Beenden</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="268"/>
<location filename="../../mainwindow.py" line="302"/>
<source>Save</source>
<translation>Speichern</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="269"/>
<location filename="../../mainwindow.py" line="303"/>
<source>Discard Changes</source>
<translation>Änderungen Verwerfen</translation>
</message>
@ -305,7 +305,7 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<message>
<location filename="../../designer/mainwindow.py" line="395"/>
<source>Abort</source>
<translation>Abbrechen</translation>
<translation type="obsolete">Abbrechen</translation>
</message>
<message>
<location filename="../../designer/mainwindow.py" line="396"/>
@ -397,6 +397,11 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<source>Save and Clone under different name</source>
<translation>Speichern und mit anderem Namen neu öffnen</translation>
</message>
<message>
<location filename="../../designer/mainwindow.py" line="395"/>
<source>Close without Save (&quot;Abort&quot;)</source>
<translation>Schließen ohne zu Speichern (&quot;Abbrechen&quot;)</translation>
</message>
</context>
<context>
<name>NewSession</name>
@ -478,13 +483,11 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<message>
<location filename="../../opensessioncontroller.py" line="86"/>
<source></source>
<translatorcomment></translatorcomment>
<translation></translation>
</message>
<message>
<location filename="../../opensessioncontroller.py" line="88"/>
<source></source>
<translatorcomment></translatorcomment>
<translation></translation>
</message>
</context>
@ -555,72 +558,72 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<context>
<name>SessionTree</name>
<message>
<location filename="../../sessiontreecontroller.py" line="144"/>
<location filename="../../sessiontreecontroller.py" line="145"/>
<source>Name</source>
<translation>Name</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="145"/>
<location filename="../../sessiontreecontroller.py" line="146"/>
<source>Last Save</source>
<translation>Letzte Speicherung</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="146"/>
<location filename="../../sessiontreecontroller.py" line="147"/>
<source>Clients</source>
<translation>Clients</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="147"/>
<location filename="../../sessiontreecontroller.py" line="148"/>
<source>Size</source>
<translation>Größe</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="148"/>
<location filename="../../sessiontreecontroller.py" line="149"/>
<source>Symlinks</source>
<translation>Symlinks</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="149"/>
<location filename="../../sessiontreecontroller.py" line="150"/>
<source>Path</source>
<translation>Pfad</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="246"/>
<location filename="../../sessiontreecontroller.py" line="247"/>
<source>About to delete Session {}</source>
<translation>Session {} soll gelöscht werden</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="245"/>
<location filename="../../sessiontreecontroller.py" line="246"/>
<source>All files in the project directory will be irreversibly deleted.</source>
<translation>Alle Dateien aus diesem Projektverzeichnis werden unwiederbringlich gelöscht.</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="254"/>
<location filename="../../sessiontreecontroller.py" line="255"/>
<source>Keep Session</source>
<translation>Session behalten</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="255"/>
<location filename="../../sessiontreecontroller.py" line="256"/>
<source>Delete!</source>
<translation>Löschen!</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="271"/>
<location filename="../../sessiontreecontroller.py" line="272"/>
<source>Copy Session</source>
<translation>Session kopieren</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="274"/>
<location filename="../../sessiontreecontroller.py" line="275"/>
<source>Force Lock Removal</source>
<translation>Lockdatei Aufhebung erzwingen</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="276"/>
<location filename="../../sessiontreecontroller.py" line="277"/>
<source>Rename Session</source>
<translation>Session umbennen</translation>
</message>
<message>
<location filename="../../sessiontreecontroller.py" line="278"/>
<location filename="../../sessiontreecontroller.py" line="279"/>
<source>Delete Session</source>
<translation>Session löschen</translation>
</message>
@ -628,22 +631,22 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<context>
<name>TrayIcon</name>
<message>
<location filename="../../systemtray.py" line="68"/>
<location filename="../../systemtray.py" line="64"/>
<source>Hide/Show Argodejo</source>
<translation>Verstecke/Zeige Argodejo</translation>
</message>
<message>
<location filename="../../systemtray.py" line="81"/>
<location filename="../../systemtray.py" line="77"/>
<source>Save &amp;&amp; Quit Argodejo</source>
<translation>Speichern und Argodejo Beenden</translation>
</message>
<message>
<location filename="../../systemtray.py" line="82"/>
<location filename="../../systemtray.py" line="78"/>
<source>Abort &amp;&amp; Quit Argodejo</source>
<translation>Abbrechen und Argodejo Beenden</translation>
</message>
<message>
<location filename="../../systemtray.py" line="88"/>
<location filename="../../systemtray.py" line="84"/>
<source>Quit </source>
<translation>Beenden </translation>
</message>
@ -651,17 +654,17 @@ Für Notizen, TODO, Referenzen, Quellen etc…</translation>
<context>
<name>mainWindow</name>
<message>
<location filename="../../mainwindow.py" line="157"/>
<location filename="../../mainwindow.py" line="188"/>
<source>Argodejo ready</source>
<translation>Argodejo ist bereit</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="192"/>
<location filename="../../mainwindow.py" line="223"/>
<source>Another GUI tried to launch.</source>
<translation>Es wurde versucht eine weitere GUI zu starten.</translation>
</message>
<message>
<location filename="../../mainwindow.py" line="219"/>
<location filename="../../mainwindow.py" line="251"/>
<source>Updating Program Database.
Thank you for your patience.</source>
<translation>Programmdatenbank wird aktualisiert.

11
qtgui/waitdialog.py

@ -54,13 +54,17 @@ class WaitDialog(object):
def __init__(self, mainWindow, text, longRunningFunction):
super().__init__()
self.text = text
logger.info(f"Starting blocking message for {longRunningFunction}")
self.mainWindow = mainWindow
self.mainWindow.ui.messageLabel.setText(text)
self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(1) #1 is messageLabel 0 is the tab widget
self.mainWindow.ui.menubar.setEnabled(False) #TODO: this will leave the options in the TrayIcon menu available.. but well, who cares...
wt = WaitThread(mainWindow, longRunningFunction)
def wrap():
longRunningFunction(self.progressInfo)
wt = WaitThread(mainWindow, wrap)
#wt.finished.connect(self.threadDone) #does NOT trigger
wt.start()
while not wt.finished:
@ -69,4 +73,7 @@ class WaitDialog(object):
self.mainWindow.ui.menubar.setEnabled(True)
self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(0) #1 is messageLabel 0 is the tab widget
def progressInfo(self, path:str):
self.mainWindow.ui.messageLabel.setText(self.text + "\n\n" + path)

Loading…
Cancel
Save