Browse Source

zipapp works. figuring out module paths

master
Nils 4 years ago
parent
commit
4832180e97
  1. 36
      template/Makefile.in
  2. 2
      template/configure.template
  3. 2
      template/gitignore.template
  4. 66
      template/qtgui/nsmsingleserver.py
  5. 88
      template/start.py

36
template/Makefile.in

@ -9,43 +9,43 @@
all: | calfbox
#Our Program
mkdir build
mkdir -p build
cd build && printf "prefix = \"$(PREFIX)\"" > compiledprefix.py
cp -r patroneo __main__.py engine qtgui site-packages template build
#We only need the installed calfbox in local site-packages. The repo in template is full with build data anyway, don't zip that in.
cp -r patroneo __main__.py engine qtgui sitepackages template build
#We only need the installed calfbox in local sitepackages. The repo in template is full with build data anyway, don't zip that in.
rm -rf build/template/calfbox
#Clean all pycache in build
cd build && find . -type d -name "__pycache__" -exec rm -r {} +
python -m zipapp "build" --output="$(PROGRAM).bin" --python="/usr/bin/env python3"
rm compiledprefix.py
rm build/compiledprefix.py
#A mode that just compiles calfbox locally so you can run the whole program standalone
calfbox:
mkdir -p site-packages
mkdir -p sitepackages
#First build the shared lib. Instead of running make install we create the lib ourselves directly
cd template/calfbox && echo $(shell pwd)
#cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../site-packages
#cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../sitepackages
cd template/calfbox && make
cp template/calfbox/.libs/libcalfbox.so.0.0.0 site-packages/"lib$(PROGRAM).so.$(VERSION)"
cp template/calfbox/.libs/libcalfbox.so.0.0.0 sitepackages/"lib$(PROGRAM).so.$(VERSION)"
#We need to be in the directory, make uses subshells which will forget the work-dir in the next line. So here is a trick:
#cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../site-packages
#cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../sitepackages
#The line above is too much for our specialized use-case. We just copy the few files we need manually.
mkdir -p site-packages/calfbox
cp template/calfbox/py/cbox.py site-packages/calfbox
cp template/calfbox/py/_cbox2.py site-packages/calfbox
cp template/calfbox/py/__init__.py site-packages/calfbox
cp template/calfbox/py/metadata.py site-packages/calfbox
cp template/calfbox/py/sfzparser.py site-packages/calfbox
cp template/calfbox/py/nullbox.py site-packages/calfbox
mkdir -p sitepackages/calfbox
cp template/calfbox/py/cbox.py sitepackages/calfbox
cp template/calfbox/py/_cbox2.py sitepackages/calfbox
cp template/calfbox/py/__init__.py sitepackages/calfbox
cp template/calfbox/py/metadata.py sitepackages/calfbox
cp template/calfbox/py/sfzparser.py sitepackages/calfbox
cp template/calfbox/py/nullbox.py sitepackages/calfbox
clean:
cd template/calfbox && make distclean && rm -rf build
rm -rf build/
rm -rf site-packages
rm -rf sitepackages
rm -f "$(PROGRAM).bin"
rm -rf "$(PROGRAM).build"
rm Makefile
find . -type d -name "__pycache__" -exec rm -r {} +
#Convenience function for developing, not used for the build or install process
gitclean:
@ -76,7 +76,7 @@ install:
done
install -D -m 644 desktop/images/256x256.png $(DESTDIR)$(PREFIX)/share/pixmaps/$(PROGRAM).png
install -D -m 755 site-packages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM)
install -D -m 755 sitepackages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM)
install -d $(DESTDIR)$(PREFIX)/share/$(PROGRAM)
cp -r engine/resources/* $(DESTDIR)$(PREFIX)/share/$(PROGRAM)/

2
template/configure.template

@ -49,7 +49,7 @@ if version_gt $required_version_pyqt $PYQTVERSION; then echo "PyQt must be versi
echo "Sub-Configure for calfbox"
set -e
#We need to be in the directory,
cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../site-packages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null
cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../sitepackages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null
cd ../..
echo "generating makefile"

2
template/gitignore.template

@ -110,7 +110,7 @@ venv.bak/
build/
Makefile
compiledprefix.py
site-packages
sitepackages
template/calfbox/.deps
template/calfbox/build
template/calfbox/autom4te.cache

66
template/qtgui/nsmsingleserver.py

@ -27,93 +27,93 @@ from threading import Thread
from sys import argv
from .nsmclient import _IncomingMessage, _OutgoingMessage
class NSMProtocol(asyncio.DatagramProtocol):
directory = None
class NSMProtocol(asyncio.DatagramProtocol):
directory = None
addr = None #a cache
def __init__(self):
super().__init__()
def __init__(self):
super().__init__()
def connection_made(self, transport):
self.transport = transport
def datagram_received(self, data, addr):
self.transport = transport
def datagram_received(self, data, addr):
NSMProtocol.addr = addr
msg = _IncomingMessage(data)
msg = _IncomingMessage(data)
if msg.oscpath == "/nsm/server/announce":
application_name, capabilities, executable_name, api_version_major, api_version_minor, pid = msg.params
NSMProtocol.pid = pid
reply = _OutgoingMessage("/reply")
reply = _OutgoingMessage("/reply")
reply.add_arg("/nsm/server/announce")
reply.add_arg("Welcome!")
reply.add_arg("Fake Save Server")
reply.add_arg("server-control:")
self.send(reply, addr)
#['/home/user/NSM Sessions/dev-example/QtCboxNsm Exämple ツ.nXDBM', 'dev-example', 'QtCboxNsm Exämple ツ.nXDBM']
openMsg = reply = _OutgoingMessage("/nsm/client/open")
openMsg = reply = _OutgoingMessage("/nsm/client/open")
openMsg.add_arg(NSMProtocol.directory)
openMsg.add_arg("NOT-A-SESSION")
openMsg.add_arg(application_name)
self.send(openMsg, addr)
self.send(_OutgoingMessage("/nsm/client/show_optional_gui"), addr)
elif msg.oscpath == "/nsm/gui/client/save":
elif msg.oscpath == "/nsm/gui/client/save":
self.send(_OutgoingMessage("/nsm/client/save"), addr)
elif msg.oscpath == "/nsm/client/gui_is_hidden":
elif msg.oscpath == "/nsm/client/gui_is_hidden":
os.kill(NSMProtocol.pid, SIGTERM)
elif msg.oscpath == "/nsm/server/stop":
elif msg.oscpath == "/nsm/server/stop":
os.kill(NSMProtocol.pid, SIGTERM)
#else:
# print (msg.oscpath, msg.params)
def send(self, message, addr):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.sendto(message.build(), addr)
@staticmethod
def staticSave(*args):
NSMProtocol.send(None, _OutgoingMessage("/nsm/client/save"), NSMProtocol.addr)
def startSingleNSMServer(directory):
def startSingleNSMServer(directory):
"""Set all paths like NSM would receive them and nsmclient.py expects them."""
serverSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
serverSock.bind(('', 0)) # Bind to a free port provided by the host.
SERVER_PORT = serverSock.getsockname()[1]
NSMProtocol.directory = directory
serverSock.close()
serverSock.close()
os.environ["NSM_URL"] = f"osc.udp://localhost:{SERVER_PORT}/"
executableName = os.path.basename(argv[0])
executableDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
assert os.path.exists(os.path.join(executableDir, executableName))
##assert os.path.exists(os.path.join(executableDir, executableName)) not valid anymore with zipapp. But it worked for years, so I guess the code is ok.
argv[0] = os.path.join(executableDir, executableName) #NSM speciality. nsmclient exlicitely checks for this
os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir
os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir
#print (argv[0])
#print (executableName)
#print (executableDir)
#print(os.environ["PATH"])
#print(os.environ["NSM_URL"])
#loop = asyncio.get_event_loop()
#loop.create_task(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
#loop.run_forever()
#asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
#asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
logger.info(f"Starting fake NSM server on port {SERVER_PORT}")
#For Carla:
signal(SIGUSR1, NSMProtocol.staticSave)
loop = asyncio.get_event_loop()
def run_loop(loop):
asyncio.set_event_loop(loop)
t = loop.create_datagram_endpoint(NSMProtocol,local_addr=('127.0.0.1',SERVER_PORT), family=socket.AF_INET)
loop.run_until_complete(t)
loop.run_forever()
loop.run_forever()
Thread(target=lambda: run_loop(loop), daemon=True).start() #Daemon makes the thread just stop when main thread ends.

88
template/start.py

@ -76,6 +76,20 @@ import os.path
from PyQt5.QtWidgets import QApplication, QStyleFactory
from PyQt5 import QtGui
import inspect
def get_script_dir(follow_symlinks=True):
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
return os.path.dirname(path)
logger.info(f"Script dir: {get_script_dir()}")
logger.info(f"Python Version {sys.version}")
try:
@ -89,6 +103,7 @@ logger.info("Compiled version: {}".format(compiledVersion))
cboxSharedObjectVersionedName = "lib"+METADATA["shortName"]+".so." + METADATA["version"]
#ZippApp with compiledprefix.py
if compiledVersion:
PATHS={ #this gets imported
"root": "",
@ -103,21 +118,22 @@ if compiledVersion:
cboxSharedObjectPath = os.path.join(prefix, "lib", METADATA["shortName"], cboxSharedObjectVersionedName)
_root = os.path.dirname(__file__)
_root = os.path.abspath(os.path.join(_root, ".."))
fallback_cboxSharedObjectPath = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)
#Local version has higher priority
if os.path.exists(fallback_cboxSharedObjectPath): #we are not yet installed, look in the source site-packages dir
os.environ["CALFBOXLIBABSPATH"] = fallback_cboxSharedObjectPath
elif os.path.exists(cboxSharedObjectPath): #we are installed
os.environ["CALFBOXLIBABSPATH"] = cboxSharedObjectPath
else:
pass
#no support for system-wide cbox in compiled mode. Error handling at the bottom of the file
import zipfile
import tempfile
logger.info("Extracting shared library to temporary directory")
zipfilePath = get_script_dir().rstrip("/template")
assert zipfile.is_zipfile(zipfilePath), (zipfilePath) #in our tests this worked. but in lss this results not in a zip file header. linux file also says it is no zip. However, unzip works.
#Extract included .so to tmp dir, tmp dir gets garbage collected at the end of our program.
libsharedDir = tempfile.TemporaryDirectory()
with zipfile.ZipFile(zipfilePath, mode="r") as ourzipappfile:
ourzipappfile.extract(f"sitepackages/{cboxSharedObjectVersionedName}", path=libsharedDir.name)
cboxso = os.path.join(libsharedDir.name, f"sitepackages/{cboxSharedObjectVersionedName}")
logger.info(f"Shared library extracted to: {cboxso}")
os.environ["CALFBOXLIBABSPATH"] = cboxso
#Not compiled, not installed. Running pure python directly in the source tree.
else:
_root = os.path.dirname(__file__)
_root = os.path.abspath(os.path.join(_root, ".."))
@ -130,14 +146,14 @@ else:
"templateShare": os.path.join(_root, "template", "engine", "resources"),
#"lib": "", #use only system paths
}
if os.path.exists (os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)):
os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)
if os.path.exists (os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName)):
os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName)
#else use system-wide.
if os.path.exists (os.path.join(_root, "site-packages", "calfbox", "cbox.py")):
#add to the front to have higher priority than system site-packages
logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "site-packages", "calfbox", "cbox.py")))
sys.path.insert(0, os.path.join(os.path.join(_root, "site-packages")))
if os.path.exists (os.path.join(_root, "sitepackages", "calfbox", "cbox.py")):
#add to the front to have higher priority than system sitepackages
logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "sitepackages", "calfbox", "cbox.py")))
sys.path.insert(0, os.path.join(os.path.join(_root, "sitepackages")))
#else try to use system-wide calfbox. Check for this and if the .so exists at the end of this file.
@ -298,7 +314,7 @@ def startPseudoNSMServer(path):
from .qtgui.nsmsingleserver import startSingleNSMServer
startSingleNSMServer(path) #provides NSM_URL environment variable and a limited drop-in replacement for NSM that will only answer to our application
assert getenv("NSM_URL")
sys.path.append("site-packages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir site-packages. Add path to the last place, in case there is an installed or bundled version
sys.path.append("sitepackages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir sitepackages. Add path to the last place, in case there is an installed or bundled version
if args.directory:
#Switch to the mode without NSM.
@ -344,20 +360,24 @@ if "CALFBOXLIBABSPATH" in os.environ:
else:
logger.info("Looking for calfbox shared library systemwide through ctypes.util.find_library")
try:
from calfbox import cbox
logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__)))
except Exception as e:
print (e)
print ("Here is some information. Please show this to the developers.")
if "calfbox" in sys.modules:
print (sys.modules["calfbox"], "->", os.path.abspath(sys.modules["calfbox"].__file__))
else:
print ("calfbox python module is not in sys.modules. This means it truly can't be found or you forgot --mute")
print ("sys.path start and tail:", sys.path[0:5], sys.path[-1])
exitWithMessage("Calfbox module could not be loaded")
if compiledVersion:
from sitepackages import cbox
logger.info(f"Calbox Python module loaded: {os.path.abspath(cbox.__file__)}")
else:
try:
from calfbox import cbox
logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__)))
except Exception as e:
print (e)
print ("Here is some information. Please show this to the developers.")
if "calfbox" in sys.modules:
print (sys.modules["calfbox"], "->", os.path.abspath(sys.modules["calfbox"].__file__))
else:
print ("calfbox python module is not in sys.modules. This means it truly can't be found or you forgot --mute")
print ("sys.path start and tail:", sys.path[0:5], sys.path[-1])
exitWithMessage("Calfbox module could not be loaded")
#Capture Ctlr+C / SIGINT and let @atexit handle the rest.

Loading…
Cancel
Save