Browse Source

replace nuitka with python builtin zipapp, removing one unreliable dependency

master
Nils 2 years ago
parent
commit
27eeb9adef
  1. 3
      template/.gitignore
  2. 48
      template/Makefile.in
  3. 9
      template/configure.template
  4. 3
      template/documentation/readme.template
  5. 7
      template/gitignore.template
  6. 66
      template/qtgui/nsmsingleserver.py
  7. 80
      template/start.py

3
template/.gitignore

@ -104,9 +104,10 @@ venv.bak/
.mypy_cache/
#nuitka and makefile
#Build
*.bin
*.build
build/
Makefile
site-packages
calfbox/.deps

48
template/Makefile.in

@ -9,36 +9,44 @@
all: | calfbox
#Our Program
printf "prefix = \"$(PREFIX)\"" > compiledprefix.py
PYTHONPATH="site-packages" python3 -m nuitka --recurse-all --python-flag -O --warn-unusual-code --warn-implicit-exceptions --recurse-not-to=PyQt5 --show-progress --show-modules --file-reference-choice=runtime $(PROGRAM)
rm compiledprefix.py
#A mode that just compiles calfbox locally so you can run the whole program standalone without nuitka
mkdir -p build
cd build && printf "prefix = \"$(PREFIX)\"" > compiledprefix.py
cp -r patroneo __main__.py engine qtgui sitepackages template build
#We only need the installed calfbox in local sitepackages. The repo in template is full with build data anyway, don't zip that in.
rm -rf build/template/calfbox
#Clean all pycache in build
cd build && find . -type d -name "__pycache__" -exec rm -r {} +
python -m zipapp "build" --output="$(PROGRAM).bin" --python="/usr/bin/env python3"
rm build/compiledprefix.py
#A mode that just compiles calfbox locally so you can run the whole program standalone
calfbox:
mkdir -p site-packages
mkdir -p sitepackages
#First build the shared lib. Instead of running make install we create the lib ourselves directly
cd template/calfbox && echo $(shell pwd)
#cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../site-packages
#cd template/calfbox && make && make install DESTDIR=$(shell pwd)/../../sitepackages
cd template/calfbox && make
cp template/calfbox/.libs/libcalfbox.so.0.0.0 site-packages/"lib$(PROGRAM).so.$(VERSION)"
cp template/calfbox/.libs/libcalfbox.so.0.0.0 sitepackages/"lib$(PROGRAM).so.$(VERSION)"
#We need to be in the directory, make uses subshells which will forget the work-dir in the next line. So here is a trick:
#cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../site-packages
#cd template/calfbox && python3 setup.py build && python3 setup.py install --user --install-lib ../../sitepackages
#The line above is too much for our specialized use-case. We just copy the few files we need manually.
mkdir -p site-packages/calfbox
cp template/calfbox/py/cbox.py site-packages/calfbox
cp template/calfbox/py/_cbox2.py site-packages/calfbox
cp template/calfbox/py/__init__.py site-packages/calfbox
cp template/calfbox/py/metadata.py site-packages/calfbox
cp template/calfbox/py/sfzparser.py site-packages/calfbox
cp template/calfbox/py/nullbox.py site-packages/calfbox
mkdir -p sitepackages/calfbox
cp template/calfbox/py/cbox.py sitepackages/calfbox
cp template/calfbox/py/_cbox2.py sitepackages/calfbox
cp template/calfbox/py/__init__.py sitepackages/calfbox
cp template/calfbox/py/metadata.py sitepackages/calfbox
cp template/calfbox/py/sfzparser.py sitepackages/calfbox
cp template/calfbox/py/nullbox.py sitepackages/calfbox
clean:
cd template/calfbox && make distclean && rm -rf build
rm -rf site-packages
cd template/calfbox && rm -rf .deps/ Makefile.in aclocal.m4 autom4te.cache/ compile config.guess config.h.in config.h.in~ config.sub configure depcomp install-sh ltmain.sh missing
rm -rf build/
rm -rf sitepackages
rm -f "$(PROGRAM).bin"
rm -rf "$(PROGRAM).build"
rm Makefile
find . -type d -name "__pycache__" -exec rm -r {} +
#Convenience function for developing, not used for the build or install process
gitclean:
@ -57,7 +65,7 @@ install:
install -D -m 644 README.md $(DESTDIR)$(PREFIX)/share/doc/$(PROGRAM)/README.md
install -D -m 644 LICENSE $(DESTDIR)$(PREFIX)/share/doc/$(PROGRAM)/LICENSE
install -D -m 644 desktop/desktop.desktop $(DESTDIR)$(PREFIX)/share/applications/org.laborejo.$(PROGRAM).desktop
install -D -m 644 desktop/desktop.desktop $(DESTDIR)$(PREFIX)/share/applications/org.laborejo.$(PROGRAM).desktop
install -d $(DESTDIR)$(PREFIX)/share/man/man1
gzip -c documentation/$(PROGRAM).1 > $(DESTDIR)$(PREFIX)/share/man/man1/$(PROGRAM).1.gz
@ -69,7 +77,7 @@ install:
done
install -D -m 644 desktop/images/256x256.png $(DESTDIR)$(PREFIX)/share/pixmaps/$(PROGRAM).png
install -D -m 755 site-packages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM)
install -D -m 755 sitepackages/lib$(PROGRAM).so.$(VERSION) -t $(DESTDIR)$(PREFIX)/lib/$(PROGRAM)
install -d $(DESTDIR)$(PREFIX)/share/$(PROGRAM)
cp -r engine/resources/* $(DESTDIR)$(PREFIX)/share/$(PROGRAM)/

9
template/configure.template

@ -7,7 +7,6 @@
prefix=/usr/local
required_version_python=3.6
required_version_pyqt=5.0
required_version_nuitka=0.6
for arg in "$@"; do
@ -24,7 +23,7 @@ for arg in "$@"; do
--help)
echo 'usage: ./configure [options]'
echo 'options:'
echo ' --prefix=<path>: installation prefix'
echo ' --prefix=<path>: installation prefix'
#echo ' --enable-debug: include debug symbols'
#echo ' --disable-debug: do not include debug symbols'
echo 'all invalid options are silently ignored'
@ -47,14 +46,10 @@ python3 -c 'import PyQt5' >/dev/null 2>&1 || { echo >&2 "PyQt for Python3 >= $re
PYQTVERSION=$(python3 -c 'from PyQt5.QtCore import QT_VERSION_STR; print(QT_VERSION_STR)')
if version_gt $required_version_pyqt $PYQTVERSION; then echo "PyQt must be version >= $required_version_pyqt but is $PYQTVERSION. Aborting."; exit 1; fi
command -v nuitka3 >/dev/null 2>&1 || { echo >&2 "Nuitka3 is required but it's not installed. Aborting."; exit 1; }
NUITKAVERSION=$(python3 -c 'import nuitka.Version; print(nuitka.Version.getNuitkaVersion())')
if version_gt $required_version_nuitka $NUITKAVERSION; then echo "Nuitka3 must be version >= $required_version_nuitka but is $NUITKAVERSION. Aborting."; exit 1; fi
echo "Sub-Configure for calfbox"
set -e
#We need to be in the directory,
cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../site-packages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null
cd template/calfbox && ./autogen.sh && ./configure --prefix=$(pwd)/../../sitepackages --without-ncurses --without-python --without-libusb $cboxconfigure > /dev/null
cd ../..
echo "generating makefile"

3
template/documentation/readme.template

@ -37,7 +37,6 @@ It is possible to clone a git repository.
#### Build Dependencies
* Bash
* Nuitka >= 0.6 (maybe earlier. Optional when running from source dir)
* GCC (development is done on 8.2, but most likely you can use a much earlier version)
### Environment:
@ -90,7 +89,7 @@ You can run <name> after extracting the release archive or cloning from git, wit
### Calfbox
"Calfbox" is the name of our internal realtime midi/audio python module.
* You can either choose to install the module systemwide, which will make running all Laborejo Software Suite programs more convenient (when run from the source dir). Please consult https://github.com/kfoltman/calfbox
* You can either choose to install the module systemwide, which will make running all Laborejo Software Suite programs more convenient (when run from the source dir). Please consult https://github.com/kfoltman/calfbox
* Or you just run `./configure` and `make calfbox` without subsequent install, which creates a `site-packages` directory in the source dir.
* A third option is `<shortname> --mute` which runs without sound at all and does not need calfbox.

7
template/gitignore.template

@ -104,12 +104,13 @@ venv.bak/
.mypy_cache/
#nuitka and makefile
#build process
*.bin
*.build
build/
Makefile
engine/compiledprefix.py
site-packages
compiledprefix.py
sitepackages
template/calfbox/.deps
template/calfbox/build
template/calfbox/autom4te.cache

66
template/qtgui/nsmsingleserver.py

@ -27,93 +27,93 @@ from threading import Thread
from sys import argv
from .nsmclient import _IncomingMessage, _OutgoingMessage
class NSMProtocol(asyncio.DatagramProtocol):
directory = None
class NSMProtocol(asyncio.DatagramProtocol):
directory = None
addr = None #a cache
def __init__(self):
super().__init__()
def __init__(self):
super().__init__()
def connection_made(self, transport):
self.transport = transport
def datagram_received(self, data, addr):
self.transport = transport
def datagram_received(self, data, addr):
NSMProtocol.addr = addr
msg = _IncomingMessage(data)
msg = _IncomingMessage(data)
if msg.oscpath == "/nsm/server/announce":
application_name, capabilities, executable_name, api_version_major, api_version_minor, pid = msg.params
NSMProtocol.pid = pid
reply = _OutgoingMessage("/reply")
reply = _OutgoingMessage("/reply")
reply.add_arg("/nsm/server/announce")
reply.add_arg("Welcome!")
reply.add_arg("Fake Save Server")
reply.add_arg("server-control:")
self.send(reply, addr)
#['/home/user/NSM Sessions/dev-example/QtCboxNsm Exämple ツ.nXDBM', 'dev-example', 'QtCboxNsm Exämple ツ.nXDBM']
openMsg = reply = _OutgoingMessage("/nsm/client/open")
openMsg = reply = _OutgoingMessage("/nsm/client/open")
openMsg.add_arg(NSMProtocol.directory)
openMsg.add_arg("NOT-A-SESSION")
openMsg.add_arg(application_name)
self.send(openMsg, addr)
self.send(_OutgoingMessage("/nsm/client/show_optional_gui"), addr)
elif msg.oscpath == "/nsm/gui/client/save":
elif msg.oscpath == "/nsm/gui/client/save":
self.send(_OutgoingMessage("/nsm/client/save"), addr)
elif msg.oscpath == "/nsm/client/gui_is_hidden":
elif msg.oscpath == "/nsm/client/gui_is_hidden":
os.kill(NSMProtocol.pid, SIGTERM)
elif msg.oscpath == "/nsm/server/stop":
elif msg.oscpath == "/nsm/server/stop":
os.kill(NSMProtocol.pid, SIGTERM)
#else:
# print (msg.oscpath, msg.params)
def send(self, message, addr):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.sendto(message.build(), addr)
@staticmethod
def staticSave(*args):
NSMProtocol.send(None, _OutgoingMessage("/nsm/client/save"), NSMProtocol.addr)
def startSingleNSMServer(directory):
def startSingleNSMServer(directory):
"""Set all paths like NSM would receive them and nsmclient.py expects them."""
serverSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
serverSock.bind(('', 0)) # Bind to a free port provided by the host.
SERVER_PORT = serverSock.getsockname()[1]
NSMProtocol.directory = directory
serverSock.close()
serverSock.close()
os.environ["NSM_URL"] = f"osc.udp://localhost:{SERVER_PORT}/"
executableName = os.path.basename(argv[0])
executableDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
assert os.path.exists(os.path.join(executableDir, executableName))
##assert os.path.exists(os.path.join(executableDir, executableName)) not valid anymore with zipapp. But it worked for years, so I guess the code is ok.
argv[0] = os.path.join(executableDir, executableName) #NSM speciality. nsmclient exlicitely checks for this
os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir
os.environ["PATH"] = os.environ["PATH"] + ":" + executableDir
#print (argv[0])
#print (executableName)
#print (executableDir)
#print(os.environ["PATH"])
#print(os.environ["NSM_URL"])
#loop = asyncio.get_event_loop()
#loop.create_task(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
#loop.run_forever()
#asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
#asyncio.run(asyncio.start_server(handle_client, 'localhost', SERVER_PORT))
logger.info(f"Starting fake NSM server on port {SERVER_PORT}")
#For Carla:
signal(SIGUSR1, NSMProtocol.staticSave)
loop = asyncio.get_event_loop()
def run_loop(loop):
asyncio.set_event_loop(loop)
t = loop.create_datagram_endpoint(NSMProtocol,local_addr=('127.0.0.1',SERVER_PORT), family=socket.AF_INET)
loop.run_until_complete(t)
loop.run_forever()
loop.run_forever()
Thread(target=lambda: run_loop(loop), daemon=True).start() #Daemon makes the thread just stop when main thread ends.

80
template/start.py

@ -65,7 +65,7 @@ logger.info("import")
"""set up python search path before the program starts and cbox gets imported.
We need to be earliest, so let's put it here.
This is influence during compiling by creating a temporary file "compiledprefix.py".
Nuitka complies that in, when make is finished we delete it.
pyzipapp archives that in, when make is finished we delete it.
#Default mode is a self-contained directory relative to the uncompiled patroneo python start script
"""
@ -76,6 +76,20 @@ import os.path
from PyQt5.QtWidgets import QApplication, QStyleFactory
from PyQt5 import QtGui
import inspect
def get_script_dir(follow_symlinks=True):
if getattr(sys, 'frozen', False): # py2exe, PyInstaller, cx_Freeze
path = os.path.abspath(sys.executable)
else:
path = inspect.getabsfile(get_script_dir)
if follow_symlinks:
path = os.path.realpath(path)
return os.path.dirname(path)
logger.info(f"Script dir: {get_script_dir()}")
logger.info(f"Python Version {sys.version}")
try:
@ -89,6 +103,7 @@ logger.info("Compiled version: {}".format(compiledVersion))
cboxSharedObjectVersionedName = "lib"+METADATA["shortName"]+".so." + METADATA["version"]
#ZippApp with compiledprefix.py
if compiledVersion:
PATHS={ #this gets imported
"root": "",
@ -103,21 +118,24 @@ if compiledVersion:
cboxSharedObjectPath = os.path.join(prefix, "lib", METADATA["shortName"], cboxSharedObjectVersionedName)
_root = os.path.dirname(__file__)
_root = os.path.abspath(os.path.join(_root, ".."))
fallback_cboxSharedObjectPath = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)
#Local version has higher priority
import zipfile
import tempfile
logger.info("Extracting shared library to temporary directory")
zipfilePath = get_script_dir().rstrip("/template")
assert zipfile.is_zipfile(zipfilePath), (zipfilePath) #in our tests this worked. but in lss this results not in a zip file header. linux file also says it is no zip. However, unzip works.
#Extract included .so to tmp dir, tmp dir gets garbage collected at the end of our program.
libsharedDir = tempfile.TemporaryDirectory()
with zipfile.ZipFile(zipfilePath, mode="r") as ourzipappfile:
ourzipappfile.extract(f"sitepackages/{cboxSharedObjectVersionedName}", path=libsharedDir.name)
if os.path.exists(fallback_cboxSharedObjectPath): #we are not yet installed, look in the source site-packages dir
os.environ["CALFBOXLIBABSPATH"] = fallback_cboxSharedObjectPath
elif os.path.exists(cboxSharedObjectPath): #we are installed
os.environ["CALFBOXLIBABSPATH"] = cboxSharedObjectPath
else:
pass
#no support for system-wide cbox in compiled mode. Error handling at the bottom of the file
sys.path.append(os.path.join(zipfilePath,"sitepackages"))
cboxso = os.path.join(libsharedDir.name, f"sitepackages/{cboxSharedObjectVersionedName}")
logger.info(f"Shared library extracted to: {cboxso}")
os.environ["CALFBOXLIBABSPATH"] = cboxso
#Not compiled, not installed. Running pure python directly in the source tree.
else:
_root = os.path.dirname(__file__)
_root = os.path.abspath(os.path.join(_root, ".."))
@ -130,14 +148,14 @@ else:
"templateShare": os.path.join(_root, "template", "engine", "resources"),
#"lib": "", #use only system paths
}
if os.path.exists (os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)):
os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "site-packages", cboxSharedObjectVersionedName)
if os.path.exists (os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName)):
os.environ["CALFBOXLIBABSPATH"] = os.path.join(_root, "sitepackages", cboxSharedObjectVersionedName)
#else use system-wide.
if os.path.exists (os.path.join(_root, "site-packages", "calfbox", "cbox.py")):
#add to the front to have higher priority than system site-packages
logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "site-packages", "calfbox", "cbox.py")))
sys.path.insert(0, os.path.join(os.path.join(_root, "site-packages")))
if os.path.exists (os.path.join(_root, "sitepackages", "calfbox", "cbox.py")):
#add to the front to have higher priority than system sitepackages
logger.info("Will attempt to start with local calfbox python module: {}".format(os.path.join(_root, "sitepackages", "calfbox", "cbox.py")))
sys.path.insert(0, os.path.join(os.path.join(_root, "sitepackages")))
#else try to use system-wide calfbox. Check for this and if the .so exists at the end of this file.
@ -298,7 +316,7 @@ def startPseudoNSMServer(path):
from .qtgui.nsmsingleserver import startSingleNSMServer
startSingleNSMServer(path) #provides NSM_URL environment variable and a limited drop-in replacement for NSM that will only answer to our application
assert getenv("NSM_URL")
sys.path.append("site-packages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir site-packages. Add path to the last place, in case there is an installed or bundled version
sys.path.append("sitepackages") # If you compiled but did not install you can still run with the local build of cbox in our temp dir sitepackages. Add path to the last place, in case there is an installed or bundled version
if args.directory:
#Switch to the mode without NSM.
@ -338,25 +356,37 @@ if args.mute:
#Make sure calfbox is available.
pycboxfound = False
if "CALFBOXLIBABSPATH" in os.environ:
logger.info("Looking for calfbox shared library in absolute path: {}".format(os.environ["CALFBOXLIBABSPATH"]))
else:
logger.info("Looking for calfbox shared library systemwide through ctypes.util.find_library")
try:
from calfbox import cbox
logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__)))
except Exception as e:
print (e)
if compiledVersion and not args.mute:
try:
from sitepackages.calfbox import cbox
logger.info(f"Calbox Python module loaded: {os.path.abspath(cbox.__file__)}")
pycboxfound = True
except Exception as e:
print (e)
else:
try:
from calfbox import cbox
logger.info("{}: using cbox python module from {} . Local version has higher priority than system wide.".format(METADATA["name"], os.path.abspath(cbox.__file__)))
pycboxfound = True
except Exception as e:
print (e)
if not pycboxfound:
print ("Here is some information. Please show this to the developers.")
if "calfbox" in sys.modules:
print (sys.modules["calfbox"], "->", os.path.abspath(sys.modules["calfbox"].__file__))
else:
print ("calfbox python module is not in sys.modules. This means it truly can't be found or you forgot --mute")
print ("sys.path start and tail:", sys.path[0:5], sys.path[-1])
print ("sys.path start and tail:", sys.path[0:5], sys.path[-5:-1])
exitWithMessage("Calfbox module could not be loaded")

Loading…
Cancel
Save