From 4832180e9723fcf80a2ceda6a093f8559facab5e Mon Sep 17 00:00:00 2001 From: Nils <> Date: Fri, 1 Jan 2021 18:55:23 +0100 Subject: [PATCH] zipapp works. figuring out module paths --- template/Makefile.in | 36 ++++++------- template/configure.template | 2 +- template/gitignore.template | 2 +- template/qtgui/nsmsingleserver.py | 66 +++++++++++------------ template/start.py | 88 +++++++++++++++++++------------ 5 files changed, 107 insertions(+), 87 deletions(-) diff --git a/template/Makefile.in b/template/Makefile.in index ff1702c..74ef5a8 100644 --- a/template/Makefile.in +++ b/template/Makefile.in @@ -9,43 +9,43 @@ all: | calfbox #Our Program - mkdir build + mkdir -p build cd build && printf "prefix = \"$(PREFIX)\"" > compiledprefix.py - cp -r patroneo __main__.py engine qtgui site-packages template build - #We only need the installed calfbox in local site-packages. The repo in template is full with build data anyway, don't zip that in. + cp -r patroneo __main__.py engine qtgui sitepackages template build + #We only need the installed calfbox in local sitepackages. The repo in template is full with build data anyway, don't zip that in. rm -rf build/template/calfbox #Clean all pycache in build cd build && find . -type d -name "__pycache__" -exec rm -r {} + python -m zipapp "build" --output="$(PROGRAM).bin" --python="/usr/bin/env python3" - rm compiledprefix.py + rm build/compiledprefix.py #A mode that just compiles calfbox locally so you can run the whole program standalone calfbox: - mkdir -p site-packages + mkdir -p sitepackages #First build the shared lib. Instead of running make install we create the lib ourselves directly cd template/calfbox && echo $(shell pwd) - #cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../site-packages + #cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../sitepackages cd template/calfbox && make - cp template/calfbox/.libs/libcalfbox.so.0.0.0 site-packages/"lib$(PROGRAM).so.$(VERSION)" + cp template/calfbox/.libs/libcalfbox.so.0.0.0 sitepackages/"lib$(PROGRAM).so.$(VERSION)" #We need to be in the directory, make uses subshells which will forget the work-dir in the next line. So here is a trick: - #cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../site-packages + #cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../sitepackages #The line above is too much for our specialized use-case. We just copy the few files we need manually. - mkdir -p site-packages/calfbox - cp template/calfbox/py/cbox.py site-packages/calfbox - cp template/calfbox/py/_cbox2.py site-packages/calfbox - cp template/calfbox/py/__init__.py site-packages/calfbox - cp template/calfbox/py/metadata.py site-packages/calfbox - cp template/calfbox/py/sfzparser.py site-packages/calfbox - cp template/calfbox/py/nullbox.py site-packages/calfbox + mkdir -p sitepackages/calfbox + cp template/calfbox/py/cbox.py sitepackages/calfbox + cp template/calfbox/py/_cbox2.py sitepackages/calfbox + cp template/calfbox/py/__init__.py sitepackages/calfbox + cp template/calfbox/py/metadata.py sitepackages/calfbox + cp template/calfbox/py/sfzparser.py sitepackages/calfbox + cp template/calfbox/py/nullbox.py sitepackages/calfbox clean: cd template/calfbox && make distclean && rm -rf build - rm -rf build/ - rm -rf site-packages + rm -rf sitepackages rm -f "$(PROGRAM).bin" rm -rf "$(PROGRAM).build" rm Makefile + find . -type d -name "__pycache__" -exec rm -r {} + #Convenience function for developing, not used for the build or install process gitclean: @@ -76,7 +76,7 @@ install: done install -D -m 644 desktop/images/256x256.png $(DESTDIR)$(PREFIX)/share/pixmaps/$(PROGRAM).png - install -D -m 755 site-packages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM) + install -D -m 755 sitepackages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM) install -d $(DESTDIR)$(PREFIX)/share/$(PROGRAM) cp -r engine/resources/* $(DESTDIR)$(PREFIX)/share/$(PROGRAM)/ diff --git a/template/configure.template b/template/configure.template index 69bb1cd..079db4e 100644 --- a/template/configure.template +++ b/template/configure.template @@ -49,7 +49,7 @@ if version_gt $required_version_pyqt $PYQTVERSION; then echo "PyQt must be versi echo "Sub-Configure for calfbox" set -e #We need to be in the directory, -cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../site-packages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null +cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../sitepackages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null cd ../.. echo "generating makefile" diff --git a/template/gitignore.template b/template/gitignore.template index d7332cb..02c56ef 100644 --- a/template/gitignore.template +++ b/template/gitignore.template @@ -110,7 +110,7 @@ venv.bak/ build/ Makefile compiledprefix.py -site-packages +sitepackages template/calfbox/.deps template/calfbox/build template/calfbox/autom4te.cache diff --git a/template/qtgui/nsmsingleserver.py b/template/qtgui/nsmsingleserver.py index d14be18..9891779 100644 --- a/template/qtgui/nsmsingleserver.py +++ b/template/qtgui/nsmsingleserver.py @@ -27,93 +27,93 @@ from threading import Thread from sys import argv from .nsmclient import _IncomingMessage, _OutgoingMessage - - -class NSMProtocol(asyncio.DatagramProtocol): - directory = None + + +class NSMProtocol(asyncio.DatagramProtocol): + directory = None addr = None #a cache - - def __init__(self): - super().__init__() + + def __init__(self): + super().__init__() def connection_made(self, transport): - self.transport = transport - def datagram_received(self, data, addr): + self.transport = transport + def datagram_received(self, data, addr): NSMProtocol.addr = addr - - msg = _IncomingMessage(data) + + msg = _IncomingMessage(data) if msg.oscpath == "/nsm/server/announce": application_name, capabilities, executable_name, api_version_major, api_version_minor, pid = msg.params NSMProtocol.pid = pid - reply = _OutgoingMessage("/reply") + reply = _OutgoingMessage("/reply") reply.add_arg("/nsm/server/announce") reply.add_arg("Welcome!") reply.add_arg("Fake Save Server") reply.add_arg("server-control:") self.send(reply, addr) - + #['/home/user/NSM Sessions/dev-example/QtCboxNsm Exämple ツ.nXDBM', 'dev-example', 'QtCboxNsm Exämple ツ.nXDBM'] - openMsg = reply = _OutgoingMessage("/nsm/client/open") + openMsg = reply = _OutgoingMessage("/nsm/client/open") openMsg.add_arg(NSMProtocol.directory) openMsg.add_arg("NOT-A-SESSION") openMsg.add_arg(application_name) self.send(openMsg, addr) - + self.send(_OutgoingMessage("/nsm/client/show_optional_gui"), addr) - - elif msg.oscpath == "/nsm/gui/client/save": + + elif msg.oscpath == "/nsm/gui/client/save": self.send(_OutgoingMessage("/nsm/client/save"), addr) - elif msg.oscpath == "/nsm/client/gui_is_hidden": + elif msg.oscpath == "/nsm/client/gui_is_hidden": os.kill(NSMProtocol.pid, SIGTERM) - elif msg.oscpath == "/nsm/server/stop": + elif msg.oscpath == "/nsm/server/stop": os.kill(NSMProtocol.pid, SIGTERM) #else: # print (msg.oscpath, msg.params) - + def send(self, message, addr): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP sock.sendto(message.build(), addr) - + @staticmethod def staticSave(*args): NSMProtocol.send(None, _OutgoingMessage("/nsm/client/save"), NSMProtocol.addr) -def startSingleNSMServer(directory): +def startSingleNSMServer(directory): """Set all paths like NSM would receive them and nsmclient.py expects them.""" - + serverSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) serverSock.bind(('', 0)) # Bind to a free port provided by the host. SERVER_PORT = serverSock.getsockname()[1] NSMProtocol.directory = directory - serverSock.close() - + serverSock.close() + os.environ["NSM_URL"] = f"osc.udp://localhost:{SERVER_PORT}/" executableName = os.path.basename(argv[0]) executableDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) - assert os.path.exists(os.path.join(executableDir, executableName)) + ##assert os.path.exists(os.path.join(executableDir, executableName)) not valid anymore with zipapp. But it worked for years, so I guess the code is ok. argv[0] = os.path.join(executableDir, executableName) #NSM speciality. nsmclient exlicitely checks for this - os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir + os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir #print (argv[0]) #print (executableName) #print (executableDir) #print(os.environ["PATH"]) #print(os.environ["NSM_URL"]) - + #loop = asyncio.get_event_loop() #loop.create_task(asyncio.start_server(handle_client, 'localhost', SERVER_PORT)) #loop.run_forever() - #asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT)) - + #asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT)) + logger.info(f"Starting fake NSM server on port {SERVER_PORT}") - + #For Carla: signal(SIGUSR1, NSMProtocol.staticSave) - + loop = asyncio.get_event_loop() def run_loop(loop): asyncio.set_event_loop(loop) t = loop.create_datagram_endpoint(NSMProtocol,local_addr=('127.0.0.1',SERVER_PORT), family=socket.AF_INET) loop.run_until_complete(t) - loop.run_forever() + loop.run_forever() Thread(target=lambda: run_loop(loop), daemon=True).start() #Daemon makes the thread just stop when main thread ends. diff --git a/template/start.py b/template/start.py index 587fc99..647a745 100644 --- a/template/start.py +++ b/template/start.py @@ -76,6 +76,20 @@ import os.path from PyQt5.QtWidgets import QApplication, QStyleFactory from PyQt5 import QtGui + +import inspect +def get_script_dir(follow_symlinks=True): + if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze + path = os.path.abspath(sys.executable) + else: + path = inspect.getabsfile(get_script_dir) + if follow_symlinks: + path = os.path.realpath(path) + return os.path.dirname(path) + + +logger.info(f"Script dir: {get_script_dir()}") + logger.info(f"Python Version {sys.version}") try: @@ -89,6 +103,7 @@ logger.info("Compiled version: {}".format(compiledVersion)) cboxSharedObjectVersionedName = "lib"+METADATA["shortName"]+".so." + METADATA["version"] +#ZippApp with compiledprefix.py if compiledVersion: PATHS={ #this gets imported "root": "", @@ -103,21 +118,22 @@ if compiledVersion: cboxSharedObjectPath = os.path.join(prefix, "lib", METADATA["shortName"], cboxSharedObjectVersionedName) _root = os.path.dirname(__file__) _root = os.path.abspath(os.path.join(_root, "..")) - fallback_cboxSharedObjectPath = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName) - - #Local version has higher priority - - - if os.path.exists(fallback_cboxSharedObjectPath): #we are not yet installed, look in the source site-packages dir - os.environ["CALFBOXLIBABSPATH"] = fallback_cboxSharedObjectPath - elif os.path.exists(cboxSharedObjectPath): #we are installed - os.environ["CALFBOXLIBABSPATH"] = cboxSharedObjectPath - - else: - pass - #no support for system-wide cbox in compiled mode. Error handling at the bottom of the file - + import zipfile + import tempfile + logger.info("Extracting shared library to temporary directory") + zipfilePath = get_script_dir().rstrip("/template") + assert zipfile.is_zipfile(zipfilePath), (zipfilePath) #in our tests this worked. but in lss this results not in a zip file header. linux file also says it is no zip. However, unzip works. + #Extract included .so to tmp dir, tmp dir gets garbage collected at the end of our program. + libsharedDir = tempfile.TemporaryDirectory() + with zipfile.ZipFile(zipfilePath, mode="r") as ourzipappfile: + ourzipappfile.extract(f"sitepackages/{cboxSharedObjectVersionedName}", path=libsharedDir.name) + + cboxso = os.path.join(libsharedDir.name, f"sitepackages/{cboxSharedObjectVersionedName}") + logger.info(f"Shared library extracted to: {cboxso}") + os.environ["CALFBOXLIBABSPATH"] = cboxso + +#Not compiled, not installed. Running pure python directly in the source tree. else: _root = os.path.dirname(__file__) _root = os.path.abspath(os.path.join(_root, "..")) @@ -130,14 +146,14 @@ else: "templateShare": os.path.join(_root, "template", "engine", "resources"), #"lib": "", #use only system paths } - if os.path.exists (os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)): - os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName) + if os.path.exists (os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName)): + os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName) #else use system-wide. - if os.path.exists (os.path.join(_root, "site-packages", "calfbox", "cbox.py")): - #add to the front to have higher priority than system site-packages - logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "site-packages", "calfbox", "cbox.py"))) - sys.path.insert(0, os.path.join(os.path.join(_root, "site-packages"))) + if os.path.exists (os.path.join(_root, "sitepackages", "calfbox", "cbox.py")): + #add to the front to have higher priority than system sitepackages + logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "sitepackages", "calfbox", "cbox.py"))) + sys.path.insert(0, os.path.join(os.path.join(_root, "sitepackages"))) #else try to use system-wide calfbox. Check for this and if the .so exists at the end of this file. @@ -298,7 +314,7 @@ def startPseudoNSMServer(path): from .qtgui.nsmsingleserver import startSingleNSMServer startSingleNSMServer(path) #provides NSM_URL environment variable and a limited drop-in replacement for NSM that will only answer to our application assert getenv("NSM_URL") - sys.path.append("site-packages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir site-packages. Add path to the last place, in case there is an installed or bundled version + sys.path.append("sitepackages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir sitepackages. Add path to the last place, in case there is an installed or bundled version if args.directory: #Switch to the mode without NSM. @@ -344,20 +360,24 @@ if "CALFBOXLIBABSPATH" in os.environ: else: logger.info("Looking for calfbox shared library systemwide through ctypes.util.find_library") -try: - from calfbox import cbox - logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__))) - -except Exception as e: - print (e) - print ("Here is some information. Please show this to the developers.") - if "calfbox" in sys.modules: - print (sys.modules["calfbox"], "->", os.path.abspath(sys.modules["calfbox"].__file__)) - else: - print ("calfbox python module is not in sys.modules. This means it truly can't be found or you forgot --mute") - print ("sys.path start and tail:", sys.path[0:5], sys.path[-1]) - exitWithMessage("Calfbox module could not be loaded") +if compiledVersion: + from sitepackages import cbox + logger.info(f"Calbox Python module loaded: {os.path.abspath(cbox.__file__)}") +else: + try: + from calfbox import cbox + logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__))) + except Exception as e: + print (e) + print ("Here is some information. Please show this to the developers.") + if "calfbox" in sys.modules: + print (sys.modules["calfbox"], "->", os.path.abspath(sys.modules["calfbox"].__file__)) + else: + print ("calfbox python module is not in sys.modules. This means it truly can't be found or you forgot --mute") + + print ("sys.path start and tail:", sys.path[0:5], sys.path[-1]) + exitWithMessage("Calfbox module could not be loaded") #Capture Ctlr+C / SIGINT and let @atexit handle the rest.