#! /usr/bin/env python3 # -*- coding: utf-8 -*- """ Copyright 2021, Nils Hilbricht, Germany ( https://www.hilbricht.net ) This file is part of the Laborejo Software Suite ( https://www.laborejo.org ), This is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """ import logging; logger = logging.getLogger(__name__); logger.info("import") import pathlib import configparser import subprocess import os import stat from engine.start import PATHS import engine.findicons as findicons def nothing(*args): pass class SupportedProgramsDatabase(object): """Find all binaries with NSM support. Resources are: * Agordejo internal program list of known working programs. * Internal blacklist of known redundant programs (such as non-daw) or nonsense entries, like Agordejo itself * A search through the users path to find stray programs that contain NSM announce messages * Finally, local to a users system: User whitelist for any program, user blacklist. Those two have the highest priority. We generate the same format as configParser does with .desktop files as _sections dict. Example: { 'categories': 'AudioVideo;Audio;X-Recorders;X-Multitrack;X-Jack;', 'comment': 'Easy to use pattern sequencer for JACK and NSM', 'comment[de]': 'Einfach zu bedienender Pattern-Sequencer', 'exec': 'patroneo', 'genericname': 'Sequencer', 'icon': 'patroneo', 'name': 'Patroneo', 'startupnotify': 'false', 'terminal': 'false', 'type': 'Application', 'x-nsm-capable': 'true'} In case there is a file in PATH or database but has no .desktop we create our own entry with missing data. We add two keys ourselves: "agordejoExec" : this is the one we will send to nsmd. "agordejoIconPath" : absolute path as str if we found an icon, so that a GUI does need to search on its own """ def __init__(self): self.progressHook = nothing #prevents the initial programstart from sending meaningless messages for the cached data. Set and reverted in self.build self.grepexcluded = (pathlib.Path(PATHS["share"], "grepexcluded.txt")) #created by hand. see docstring #assert self.grepexcluded.exists() self.blacklist = ("nsmd", "non-daw", "carla", "agordejo", "adljack", "agordejo.bin") #only programs that have to do with audio and music. There is another general blacklist that speeds up discovery self.whiteList = ("thisdoesnotexisttest", "patroneo", "vico", "fluajho", "carla-rack", "carla-patchbay", "carla-jack-multi", "carla-jack-single", "ardour5", "ardour6", "nsm-data", "jackpatch", "nsm-proxy", "ADLplug", "ams", "drumkv1_jack", "synthv1_jack", "samplv1_jack", "padthv1_jack", "luppp", "non-mixer", "non-timeline", "non-sequencer", "non-midi-mapper", "non-mixer-noui", "OPNplug", "qmidiarp", "qtractor", "zynaddsubfx", "jack_mixer", "hydrogen", "mfp", "shuriken", "laborejo", "guitarix", "radium", "ray-proxy", "ray-jackpatch", "amsynth", "mamba", ) #shortcut list and programs not found by buildCache_grepExecutablePaths because they are just shellscripts and do not contain /nsm/server/announce. self.userWhitelist = () #added dynamically to morePrograms. highest priority self.userBlacklist = () #added dynamically to blacklist. highest priority self.knownDesktopFiles = { #shortcuts to the correct desktop files. Reverse lookup binary->desktop creates false entries, for example ZynAddSubFx and Carla. "zynaddsubfx": "zynaddsubfx-jack.desktop", #value will later get replaced with the .desktop entry #"carla-jack-single" : "carla.desktop", #We CANNOT add them here because both key and value must be unique and hashable. We create a reverse dict from this. #"carla-jack-patchbay" : "carla.desktop", #"carla-jack-rack" : "carla.desktop", "ams" : "ams.desktop", "amsynth" : "amsynth.desktop", } self._reverseKnownDesktopFiles = dict(zip(self.knownDesktopFiles.values(),self.knownDesktopFiles.keys())) #to lookup the exe by desktoip name self.programs = [] #list of dicts. guaranteed keys: agordejoExec, name, agordejoFullPath. And probably others, like description and version. self.nsmExecutables = set() #set of executables for fast membership, if a GUI wants to know if they are available. Needs to be build "manually" with self.programs. no auto-property for a list. at least we don't want to do the work. #.build needs to be called from the api/GUI. #self.unfilteredExecutables = self.buildCache_unfilteredExecutables() #This doesn't take too long. we can start that every time. It will get updated in build as well. self.unfilteredExecutables = None #in build() #self.build() #fills self.programs and def buildCache_grepExecutablePaths(self)->list: """return a list of executable names in the path (not the path itself) Grep explained: -s silent. No errors, eventhough subprocess uses stdout only -R recursive with symlinks. We don't want to look in subdirs because that is not allowed by PATH and nsm, but we want to follow symlinks If you have a custom user path that does not mean that all its executables will automatically show up here. They still need to contain /nsm/server/announce Your binaries will be in unfilteredExecutables though """ result = [] testpaths = os.environ["PATH"].split(os.pathsep) + ["/bin", "/sbin"] executablePaths = set([pathlib.Path(p).resolve() for p in os.environ["PATH"].split(os.pathsep)]) #resolve filters out symlinks, like arches /sbin and /bin. set() makes it unique excludeFromProcessingSet = set(self.blacklist + self.userBlacklist) whiteSet = set(self.whiteList + self.userWhitelist) excludeFromProcessingSet.update(whiteSet) for path in executablePaths: self.progressHook(f"{path}") command = f"grep --exclude-from {self.grepexcluded} -iRsnl {path} -e /nsm/server/announce" #command = f"grep -iRsnl {path} -e /nsm/server/announce" #Py>=3.7 completedProcess = subprocess.run(command, capture_output=True, text=True, shell=True) completedProcess = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) #universal_newlines is an alias for text, which was deprecated in 3.7 because text is more understandable. capture_output replaces the two PIPEs in 3.7 for fullPath in completedProcess.stdout.split(): self.progressHook(f"{fullPath}") exe = pathlib.Path(fullPath).relative_to(path) if not str(exe) in excludeFromProcessingSet: #skip over any known file, good or bad result.append((str(exe), str(fullPath))) for prg in whiteSet: self.progressHook(f"{prg}") for path in executablePaths: if pathlib.Path(path, prg).is_file(): #check if this actually exists result.append((str(prg), str(pathlib.Path(path, prg)))) break #inner loop return list(set(result)) #make unique def buildCache_DesktopEntries(self): """Go through all dirs including subdirs""" xdgPaths = ( pathlib.Path("/usr/share/applications"), pathlib.Path("/usr/local/share/applications"), pathlib.Path(pathlib.Path.home(), ".local/share/applications"), ) config = configparser.ConfigParser() allDesktopEntries = [] for basePath in xdgPaths: for f in basePath.glob('**/*'): self.progressHook(f"{f}") if f.is_file() and f.suffix == ".desktop": config.clear() try: config.read(f) entryDict = dict(config._sections["Desktop Entry"]) #Replace simple names in our shortcut list with full data if f.name in self.knownDesktopFiles.values(): key = self._reverseKnownDesktopFiles[f.name] self.knownDesktopFiles[key] = entryDict #in any case: allDesktopEntries.append(entryDict) #_sections 'DesktopEntry':{dictOfActualData) except: #any bad config means skip logger.warning(f"Bad desktop file. Skipping: {f}") return allDesktopEntries def setCache(self, cache:dict): """Qt Settings will send us this""" self.programs = cache["programs"] #list of dicts findicons.updateCache(cache["iconPaths"]) logger.info("Restoring program list from serialized cache") self.nsmExecutables = set(d["agordejoExec"] for d in self.programs) def getCache(self)->dict: """To carry the DB over restarts. Saved by Qt Settings at the moment""" cache = { "programs" : self.programs, #list of dicts "iconPaths" : findicons.getSerializedCache(), #list } return cache def build(self, progressHook=None): """Can be called at any time by the user to update after installing new programs""" if progressHook: #receives one string which indicates what files is currently parsed. #Just the pure path, this will not get translated! #The purpose is to show a "we are not frozen!" feedback to the user. #It doesn't really matter what is reported back as long as it changes often self.progressHook = progressHook logger.info("Building launcher database. This might take a minute") self.progressHook("") self.programs = self._build() #builds iconPaths as side-effect self.unfilteredExecutables = self.buildCache_unfilteredExecutables() self.nsmExecutables = set(d["agordejoExec"] for d in self.programs) self._buildWhitelist() self.progressHook("") self.progressHook = nothing logger.info("Building launcher database done.") def _exeToDesktopEntry(self, exe:str)->dict: """Assumes self.desktopEntries is up to date Convert one exe (not full path!) to one dict entry. """ if exe in self.knownDesktopFiles and type(self.knownDesktopFiles[exe]) is dict : #Shortcut through internal database entry = self.knownDesktopFiles[exe] return entry else: #Reverse Search desktop files. for entry in self.desktopEntries: if "exec" in entry and exe.lower() in entry["exec"].lower(): return entry #else: #Foor loop ended. Did not find any matching desktop file return None def _build(self): self.executables = self.buildCache_grepExecutablePaths() self.desktopEntries = self.buildCache_DesktopEntries() findicons.updateCache() leftovers = set(self.executables) matches = [] #list of dicts for exe, fullPath in self.executables: self.progressHook(f"{fullPath}") entry = self._exeToDesktopEntry(exe) if entry and type(entry) is dict: #Found match! entry["agordejoFullPath"] = fullPath #We don't want .desktop syntax like "qmidiarp %F" entry["agordejoExec"] = exe if entry["icon"]: foundIcon = findicons.findIconPath(entry["icon"]) else: foundIcon = findicons.findIconPath(entry["agordejoExec"]) if foundIcon: entry["agordejoIconPath"] = str(foundIcon[0]) #pick best resolution else: entry["agordejoIconPath"] = None matches.append(entry) try: leftovers.remove((exe,fullPath)) except KeyError: pass #Double entries like zyn-jack zyn-alsa etc. elif entry and not type(entry) is dict: #There is a strange bug I can't reproduce. At least catch it. logger.error(f"Wrong entry type: {type(entry)} for {entry}") for exe,fullPath in leftovers: pseudoEntry = {"name": exe.title(), "agordejoExec":exe, "agordejoFullPath":fullPath} matches.append(pseudoEntry) return matches def buildCache_unfilteredExecutables(self): def isexe(path): """executable by owner""" return path.is_file() and stat.S_IXUSR & os.stat(path)[stat.ST_MODE] == 64 result = [] executablePaths = [pathlib.Path(p) for p in os.environ["PATH"].split(os.pathsep)] for path in executablePaths: self.progressHook(f"{path}") result += [str(pathlib.Path(f).relative_to(path)) for f in path.glob("*") if isexe(f)] return sorted(list(set(result))) def _buildWhitelist(self): """For reliable, fast and easy selection this is the whitelist. It will be populated from a template-list of well-working clients and then all binaries not in the path are filtered out. This can be presented to the user without worries.""" #Assumes to be called only from self.build startexecutables = set(self.whiteList + self.userWhitelist) for prog in self.programs: prog["whitelist"] = prog["agordejoExec"] in startexecutables """ matches = [] for exe in startexecutables: entry = self._exeToDesktopEntry(exe) if entry: #Found match! #We don't want .desktop syntax like "qmidiarp %F" entry["agordejoExec"] = exe matches.append(entry) return matches """ programDatabase = SupportedProgramsDatabase()