Add progress updates and file integrity check for copy session

CHANGELOG

@@ -2,9 +2,12 @@
 Remove "Quick" mode.  As it turns out "Full" mode is quick enough. Port convenience features to full mode.
 Add button in session chooser for alternative access to context menu options
 Add normal "Save" to tray icon.
+Add file integrity check after copying a session
+Add progress updates to copy-session.
 Submenu in tray icon to toggle visibility of individual clients (if supported)
 Double click on a crashed clients opens it again. This was intentional so far, because a crash is special. But it will be fine...
 More programs and icons added to the internal database.
+Fix a rare crash where the hostname must be case sensitive.
 
 2021-01-15 Version 0.2.1
 Remove Nuitka as dependency. Build commands stay the same.

engine/api.py

@@ -213,6 +213,10 @@ def sessionList()->list:
     r = nsmServerControl.exportSessionsAsDicts()
     return [s["nsmSessionName"] for s in r]
 
+def requestSessionList():
+    """For the rare occasions where that is needed"""
+    callbacks._sessionsChanged() #send session list
+
 def buildSystemPrograms(progressHook=None):
     """Build a list of dicts with the .desktop files (or similar) of all NSM compatible programs
     present on the system"""
@@ -283,10 +287,14 @@ def sessionRename(nsmSessionName:str, newName:str):
     """only for non-open sessions"""
     nsmServerControl.renameSession(nsmSessionName, newName)
 
-def sessionCopy(nsmSessionName:str, newName:str):
+def sessionCopy(nsmSessionName:str, newName:str, progressHook=None):
     """Create a copy of the session. Removes the lockfile, if any.
-    Has some safeguards inside so it will not crash."""
-    nsmServerControl.copySession(nsmSessionName, newName)
+    Has some safeguards inside so it will not crash.
+
+    If progressHook is provided (e.g. by a GUI) it will be called at regular intervals
+    to inform of the copy process, or at least that it is still running.
+    """
+    nsmServerControl.copySession(nsmSessionName, newName, progressHook)
 
 def sessionOpen(nsmSessionName:str):
     """Saves the current session and loads a different existing session."""

engine/comparedirectories.py

@@ -0,0 +1,47 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Copyright 2021, Nils Hilbricht, Germany ( https://www.hilbricht.net )
+
+The Non-Session-Manager by Jonathan Moore Liles <male@tuxfamily.org>: http://non.tuxfamily.org/nsm/
+New Session Manager, by LinuxAudio.org: https://github.com/linuxaudio/new-session-manager
+With help from code fragments from https://github.com/attwad/python-osc ( DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE v2 )
+
+This file is part of the Laborejo Software Suite ( https://www.laborejo.org ).
+
+This application is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#https://stackoverflow.com/questions/24937495/how-can-i-calculate-a-hash-for-a-filesystem-directory-using-python
+
+import hashlib
+from _hashlib import HASH as Hash
+from pathlib import Path
+from typing import Union
+
+def md5_update_from_dir(directory, hash):
+    assert Path(directory).is_dir()
+    for path in sorted(Path(directory).iterdir(), key=lambda p: str(p).lower()):
+        hash.update(path.name.encode())
+        if path.is_file():
+            with open(path, "rb") as f:
+                for chunk in iter(lambda: f.read(4096), b""):
+                    hash.update(chunk)
+        elif path.is_dir():
+            hash = md5_update_from_dir(path, hash)
+    return hash
+
+
+def md5_dir(directory):
+    return md5_update_from_dir(directory, hashlib.md5()).hexdigest()

engine/nsmservercontrol.py

@@ -31,6 +31,7 @@ import socket
 from os import getenv #to get NSM env var
 from shutil import rmtree as shutilrmtree
 from shutil import copytree as shutilcopytree
+from multiprocessing import Process
 from urllib.parse import urlparse #to convert NSM env var
 import subprocess
 import atexit
@@ -39,10 +40,15 @@ import json
 from uuid import uuid4
 from datetime import datetime
 from sys import exit as sysexit
+from time import sleep
+
+#Our files
+from .comparedirectories import md5_dir
 
 def nothing(*args, **kwargs):
     pass
 
+
 class _IncomingMessage(object):
     """Representation of a parsed datagram representing an OSC message.
 
@@ -1481,9 +1487,13 @@ class NsmServerControl(object):
             tmp.rename(newPath)
             assert newPath.exists()
 
-    def copySession(self, nsmSessionName:str, newName:str):
+    def copySession(self, nsmSessionName:str, newName:str, progressHook=None):
         """Copy a whole tree. Keep symlinks as symlinks.
-        Lift lock"""
+        Lift lock.
+
+        If progressHook is provided (e.g. by a GUI) it will be called at regular intervals
+        to inform of the copy process, or at least that it is still running.
+        """
         self._updateSessionListBlocking()
 
         source = pathlib.Path(self.sessionRoot, nsmSessionName)
@@ -1501,7 +1511,54 @@ class NsmServerControl(object):
 
         #All is well.
         try:
-            shutilcopytree(source, destination, symlinks=True, dirs_exist_ok=False) #raises an error if dir already exists. But we already test above.
+            def mycopy():
+                shutilcopytree(source, destination, symlinks=True, dirs_exist_ok=False) #raises an error if dir already exists. But we already test above.
+
+            if progressHook:
+
+                def waiter(copyProcess):
+                    """Compare the final size with the current size and generate a percentage
+                    from it, which we send as progress"""
+                    sourceDirectorySize = sum(f.stat().st_size for f in source.glob('**/*') if f.is_file()) - 2048 #padded so we don't create an infinite loop from a rounding error
+                    destinationDirectorySize = sum(f.stat().st_size for f in destination.glob('**/*') if f.is_file())
+                    #destinationDirectorySize does not start at 0. the copy() function might already by running before waiter() starts.
+
+                    while destinationDirectorySize < sourceDirectorySize:
+                        if not copyProcess.is_alive():
+                            break
+                        percentString = str( int((destinationDirectorySize / sourceDirectorySize) * 100)) + "%"
+                        progressHook(percentString)
+                        sleep(0.5) #don't send too much. two times a second is plenty.
+                        #For next round
+                        destinationDirectorySize = sum(f.stat().st_size for f in destination.glob('**/*') if f.is_file())
+                """
+                #This moves both processes away from the main thread. It works, but Qt will not update anymore
+                #We need a way to just spawn one extra process and wait/processHook in the main process
+                processes = []
+                for function in (waiter, mycopy):
+                    proc = Process(target=function)
+                    proc.start()
+                    processes.append(proc)
+
+                for proc in processes:
+                    proc.join()
+                """
+                proc = Process(target=mycopy)
+                proc.start()
+                waiter(proc) #has the while loop to wait and check proc
+                proc.join() #finish
+
+                #Do a check if both dirs are equal
+
+                progressHook("Veryfying file-integrity. This may take a while...") #string gets translated in qt gui mainwindow. Don't change just this here.
+                sourceHash = md5_dir(source)
+                desinationHash = md5_dir(destination)
+                if not sourceHash == desinationHash:
+                    logger.error("ERROR! Copied session data is different from source session. Please check you data!")
+                    progressHook("ERROR! Copied session data is different from source session. Please check you data!")  #ERROR! is a keyword for the gui wait dialog to not switch away. This gets translated in the Qt GUI mainwindow. Don't change this string
+            else:
+                mycopy()
+
             self.forceLiftLock(newName)
         except Exception as e: #we don't want to crash if user tries to copy to /root or so.
             logger.error(e)

qtgui/designer/mainwindow.py

@@ -195,6 +195,9 @@ class Ui_MainWindow(object):
         self.messageLabel.setAlignment(QtCore.Qt.AlignCenter)
         self.messageLabel.setObjectName("messageLabel")
         self.verticalLayout_13.addWidget(self.messageLabel)
+        self.waitDialogErrorButton = QtWidgets.QPushButton(self.messagePage)
+        self.waitDialogErrorButton.setObjectName("waitDialogErrorButton")
+        self.verticalLayout_13.addWidget(self.waitDialogErrorButton)
         self.mainPageSwitcher.addWidget(self.messagePage)
         self.verticalLayout.addWidget(self.mainPageSwitcher)
         MainWindow.setCentralWidget(self.centralwidget)
@@ -274,7 +277,7 @@ class Ui_MainWindow(object):
         self.menubar.addAction(self.menuClientNameId.menuAction())
 
         self.retranslateUi(MainWindow)
-        self.mainPageSwitcher.setCurrentIndex(0)
+        self.mainPageSwitcher.setCurrentIndex(1)
         self.tabbyCat.setCurrentIndex(0)
         self.detailedStackedWidget.setCurrentIndex(0)
         QtCore.QMetaObject.connectSlotsByName(MainWindow)
@@ -309,6 +312,7 @@ class Ui_MainWindow(object):
         self.informationTreeWidget.setSortingEnabled(__sortingEnabled)
         self.tabbyCat.setTabText(self.tabbyCat.indexOf(self.tab_information), _translate("MainWindow", "Information"))
         self.messageLabel.setText(_translate("MainWindow", "Processing"))
+        self.waitDialogErrorButton.setText(_translate("MainWindow", "I understand that I will need to resolve this problem on my own!"))
         self.menuControl.setTitle(_translate("MainWindow", "Control"))
         self.menuSession.setTitle(_translate("MainWindow", "SessionName"))
         self.menuClientNameId.setTitle(_translate("MainWindow", "ClientNameId"))

qtgui/designer/mainwindow.ui

@@ -36,7 +36,7 @@
        <number>0</number>
       </property>
       <property name="currentIndex">
-       <number>0</number>
+       <number>1</number>
       </property>
       <widget class="QWidget" name="tabPage">
        <layout class="QVBoxLayout" name="verticalLayout_12">
@@ -487,6 +487,13 @@
           </property>
          </widget>
         </item>
+        <item>
+         <widget class="QPushButton" name="waitDialogErrorButton">
+          <property name="text">
+           <string>I understand that I will need to resolve this problem on my own!</string>
+          </property>
+         </widget>
+        </item>
        </layout>
       </widget>
      </widget>

qtgui/mainwindow.py

@@ -129,6 +129,9 @@ class MainWindow(QtWidgets.QMainWindow):
 
         logger.info(f"Program icons path: {QtGui.QIcon.themeSearchPaths()}, {QtGui.QIcon.themeName()}")
 
+        QtCore.QT_TRANSLATE_NOOP("NOOPEngineStrings", "ERROR! Copied session data is different from source session. Please check you data!")
+        QtCore.QT_TRANSLATE_NOOP("NOOPEngineStrings", "Veryfying file-integrity. This may take a while...")
+
         #Set up the user interface from Designer and other widgets
         self.ui =  Ui_MainWindow()
         self.ui.setupUi(self)
@@ -281,7 +284,7 @@ class MainWindow(QtWidgets.QMainWindow):
     def updateProgramDatabase(self):
         """Display a progress-dialog that waits for the database to be build.
         Automatically called on first start or when instructed by the user"""
-        text = QtCore.QCoreApplication.translate("mainWindow", "Updating Program Database.\nThank you for your patience.")
+        text = QtCore.QCoreApplication.translate("mainWindow", "Updating Program Database.\nThank you for your patience.\nIf progress freezes please kill and restart the whole program.")
 
         settings = QtCore.QSettings("LaborejoSoftwareSuite", METADATA["shortName"])
         settings.remove("engineCache")

qtgui/sessiontreecontroller.py

@@ -34,6 +34,7 @@ import engine.api as api
 from .helper import sizeof_fmt
 from .projectname import ProjectNameWidget
 from .projectname import NewSessionDialog
+from .waitdialog import WaitDialog
 
 class DirectoryItem(QtWidgets.QTreeWidgetItem):
     """A plain directory with no session content"""
@@ -340,9 +341,17 @@ class SessionTreeController(object):
 
     def _askForCopyAndCopy(self, nsmSessionName:str):
         """Called by button and context menu"""
+        copyString = QtCore.QCoreApplication.translate("SessionTree", "Copying {} to {}")
         widget = ProjectNameWidget(parent=self.treeWidget, startwith=nsmSessionName+"-copy")
         if widget.result:
-            api.sessionCopy(nsmSessionName, widget.result)
+            logger.info("Asking api to copy a session while waiting")
+            copyString = copyString.format(nsmSessionName, widget.result)
+            def longrunningfunction(progressHook):
+                api.sessionCopy(nsmSessionName, widget.result, progressHook)
+            diag = WaitDialog(self.mainWindow, copyString, longrunningfunction) #save in local var to keep alive
+
+        #Somehow session list is wrong, symlinks are not calculated. Force update.
+        api.requestSessionList()
 
     def _askForNameAndRenameSession(self, nsmSessionName:str):
         """Only for non-locked sessions. Context menu is only available if not locked."""

qtgui/waitdialog.py

@@ -55,12 +55,18 @@ class WaitDialog(object):
 
         super().__init__()
         self.text = text
+        self._errorText = None
         logger.info(f"Starting blocking message for {longRunningFunction}")
         self.mainWindow = mainWindow
         self.mainWindow.ui.messageLabel.setText(text)
         #self.mainWindow.ui.messageLabel.setWordWrap(True)  #qt segfault! maybe something with threads... don't care. We truncate now, see below.
         self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(1) #1 is messageLabel 0 is the tab widget
         self.mainWindow.ui.menubar.setEnabled(False) #TODO: this will leave the options in the TrayIcon menu available.. but well, who cares...
+        self.mainWindow.ui.waitDialogErrorButton.hide()
+        self.mainWindow.ui.waitDialogErrorButton.setEnabled(False)
+        self.mainWindow.ui.waitDialogErrorButton.clicked.connect(lambda: self.weAreDone()) #for some reason this does not trigger if we don't use lambda
+
+        self.buttonErrorText = QtCore.QCoreApplication.translate("WaitDialog", "Please confirm with a click on the button at the bottom.")
 
         def wrap():
             longRunningFunction(self.progressInfo)
@@ -71,11 +77,23 @@ class WaitDialog(object):
         while not wt.finished:
             self.mainWindow.qtApp.processEvents()
 
+        if self._errorText: #progressInfo activated it
+            self.mainWindow.ui.messageLabel.setText(self._errorText + "\n" + self.buttonErrorText)
+            self.mainWindow.ui.waitDialogErrorButton.show()
+            self.mainWindow.ui.waitDialogErrorButton.setEnabled(True)
+        else:
+            self.weAreDone()
+
+    def weAreDone(self):
         self.mainWindow.ui.menubar.setEnabled(True)
+        self.mainWindow.ui.waitDialogErrorButton.setEnabled(False)
+        self.mainWindow.ui.waitDialogErrorButton.hide()
         self.mainWindow.ui.mainPageSwitcher.setCurrentIndex(0) #1 is messageLabel 0 is the tab widget
 
 
-    def progressInfo(self, path:str):
-        #paths can be very long. They will destroy our complete layout if unchecked. we truncate to -80
-        self.mainWindow.ui.messageLabel.setText(self.text + "\n\n" + path[-80:])
-
+    def progressInfo(self, updateText:str):
+        """ProcessEvents is already called above in init"""
+        #updateText can be very long. They will destroy our complete layout if unchecked. we truncate to -80
+        self.mainWindow.ui.messageLabel.setText(self.text + "\n\n" + updateText[-80:])
+        if "ERROR!" in updateText:
+            self._errorText = updateText