From 341cc62fcfa7a8bddd52d7cd3a89d8472ad273b8 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 5 Dec 2023 09:28:53 -0500 Subject: [PATCH] Refs #11. Removed sftpclone, sftpclient from requirements, and moved to fabric --- create_conda_venv.bat | 8 ++ create_conda_venv.sh | 7 ++ libs/servers/WatchServer.py | 1 + libs/uploaders/SFTPUploader.py | 206 ++++++++++++++++++++------------- requirements.txt | 5 +- 5 files changed, 145 insertions(+), 82 deletions(-) create mode 100644 create_conda_venv.bat create mode 100644 create_conda_venv.sh diff --git a/create_conda_venv.bat b/create_conda_venv.bat new file mode 100644 index 0000000..91e3cee --- /dev/null +++ b/create_conda_venv.bat @@ -0,0 +1,8 @@ +@echo off +echo Create or update conda venv +call conda install -m -c conda-forge -y --copy -p venv python=3.10 +echo Activating venv +call conda activate .\venv +echo Installing requirements +call pip install -r requirements.txt +call conda deactivate diff --git a/create_conda_venv.sh b/create_conda_venv.sh new file mode 100644 index 0000000..61690dd --- /dev/null +++ b/create_conda_venv.sh @@ -0,0 +1,7 @@ +echo "Create or update conda venv" +conda install -m -c conda-forge -y --copy -p $PWD/venv python=3.10 +echo "Activating venv" +conda activate $PWD/venv +echo "Installing requirements (make sure git submodules are installed)..." +$PWD/venv/bin/pip install -r $PWD/requirements.txt +conda deactivate diff --git a/libs/servers/WatchServer.py b/libs/servers/WatchServer.py index 9c9361f..69caef3 100644 --- a/libs/servers/WatchServer.py +++ b/libs/servers/WatchServer.py @@ -69,6 +69,7 @@ def sync_files(self, check_internet: bool = True): self.synching_files = True # Build list of files to transfer base_folder = self.data_path + '/ToProcess/' + base_folder = base_folder.replace('/', os.sep) files = [] full_files = [] file_folders = [] diff --git a/libs/uploaders/SFTPUploader.py b/libs/uploaders/SFTPUploader.py index 8fc0c85..377898b 100644 --- a/libs/uploaders/SFTPUploader.py +++ b/libs/uploaders/SFTPUploader.py @@ -4,18 +4,19 @@ # Authors: Simon Brière, Eng. MASc. # Mathieu Hamel, Eng. MASc. ################################################## -import paramiko -import pysftp import logging import os +import time +import socket +import stat from os import walk -from libs.hardware.PiHubHardware import PiHubHardware +from os import makedirs -from sftpclone import sftpclone +import fabric +from paramiko import BadHostKeyException, AuthenticationException, SSHException, SFTPClient +from libs.hardware.PiHubHardware import PiHubHardware from libs.utils.Network import Network -import time -from os import makedirs class SFTPUploader: @@ -28,55 +29,57 @@ def sftp_send(sftp_config: dict, files_directory_on_server: [str], files_to_tran PiHubHardware.ensure_internet_is_available() # Do the transfer - cnopts = pysftp.CnOpts() - cnopts.hostkeys = None logging.info('About to send files to server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"])) file_to_transfer = None try: - with pysftp.Connection(host=sftp_config["hostname"], username=sftp_config["username"], - password=sftp_config["password"], port=sftp_config["port"], - cnopts=cnopts) as s: + with fabric.Connection(host=sftp_config["hostname"], user=sftp_config["username"], + connect_kwargs={'password': sftp_config["password"]}, + port=sftp_config["port"], connect_timeout=10) as c: + c.open() + client: SFTPClient = c.sftp() for file_server_location, file_to_transfer in zip(files_directory_on_server, files_to_transfer): - if not (s.isdir(file_server_location)): - s.makedirs(file_server_location) - # s.mkdir(file_server_location) - with s.cd(file_server_location): - file_name = os.path.basename(file_to_transfer) - # Query file size from server and compare to local file size - try: - remote_attr = s.stat(remotepath=file_server_location + '/' + file_name) - remote_size = remote_attr.st_size - local_attr = os.stat(file_to_transfer) - local_size = local_attr.st_size - if local_size == remote_size: - # Same size on server as local file, skip! - logging.info('Skipping ' + file_to_transfer + ': already present on server.') - if file_transferred_callback: - file_transferred_callback(file_to_transfer) - continue - except IOError: - # File not on server = ok, continue! - pass - # TODO: Use pysftp.Connection.stat() to find file size and only send if is different - s.put(localpath=file_to_transfer, preserve_mtime=True, - callback=lambda current, total: - SFTPUploader.file_upload_progress(current, total, file_to_transfer, - file_transferred_callback)) - logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...') - # s.close() - # logging.info('File ' + file_to_transfer + ' sent.') - except (pysftp.exceptions.ConnectionException, pysftp.CredentialException, - pysftp.AuthenticationException, pysftp.HostKeysException, - paramiko.SSHException, paramiko.PasswordRequiredException) as exc: + # Create directories on server if needed + if not (SFTPUploader.isdir(client, file_server_location)): + SFTPUploader.makedirs(client, file_server_location) + + # Move to directory + # client.chdir(file_server_location) + file_name = os.path.basename(file_to_transfer) + # Query file size from server and compare to local file size + local_attr = os.stat(file_to_transfer) + local_size = local_attr.st_size + remote_file_name = file_server_location + '/' + file_name + try: + remote_attr = client.lstat(remote_file_name) + remote_size = remote_attr.st_size + if local_size == remote_size: + # Same size on server as local file, skip! + logging.info('Skipping ' + file_to_transfer + ': already present on server.') + if file_transferred_callback: + file_transferred_callback(file_to_transfer) + continue + except IOError: + # File not on server = ok, continue! + pass + + logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...') + client.put(localpath=file_to_transfer, remotepath=remote_file_name, confirm=True, + callback=lambda current, total: + SFTPUploader.file_upload_progress(current, total, file_to_transfer, + file_transferred_callback)) + # Update time on server + times = (local_attr.st_atime, local_attr.st_mtime) + client.utime(remote_file_name, times) + + except (BadHostKeyException, SSHException, AuthenticationException, socket.error, IOError) as exc: err_msg = str(exc) - if exc.message: + if hasattr(exc, 'message'): err_msg = exc.message + ' ' + err_msg if file_to_transfer: - logging.error('Error occurred transferring ' + file_to_transfer + ': ' + err_msg) + logging.error('Error occurred transferring ' + str(file_to_transfer) + ': ' + err_msg) else: logging.error('Error occured while trying to transfer: ' + err_msg) return False - logging.info('Files transfer complete!') return True @@ -88,25 +91,26 @@ def sftp_merge_and_send(sftp_config: dict, file_path_on_server: str, file_server # Check if Internet connected if check_internet: PiHubHardware.ensure_internet_is_available() - cnopts = pysftp.CnOpts() - cnopts.hostkeys = None - logging.info('About to get file from server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"])) + + logging.info('About to get files from server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"])) try: - with pysftp.Connection(host=sftp_config["hostname"], username=sftp_config["username"], - password=sftp_config["password"], port=sftp_config["port"], - cnopts=cnopts) as s: + with fabric.Connection(host=sftp_config["hostname"], user=sftp_config["username"], + connect_kwargs={'password': sftp_config["password"]}, + port=sftp_config["port"], connect_timeout=10) as c: + c.open() + client: SFTPClient = c.sftp() # Check if the file exist on remote and get it locally - if not (s.isfile(file_path_on_server)): + if not (SFTPUploader.isfile(client, file_path_on_server)): logging.info('No file on server to merge with ' + file_to_transfer) else: - s.get(file_path_on_server, temporary_file) + client.get(file_path_on_server, temporary_file) # Now do the merge in the local file m_point = "/mnt/app" - lines = lambda f: [l for l in open(f).read().splitlines()] + lines = lambda f: [line for line in open(f).read().splitlines()] lines1 = lines(temporary_file) lines2 = lines(file_to_transfer) - checks = [l.split(m_point)[-1] for l in lines1] - for item in sum([[l for l in lines2 if c in l] for c in checks], []): + checks = [line.split(m_point)[-1] for line in lines1] + for item in sum([[line for line in lines2 if c in line] for c in checks], []): lines2.remove(item) if os.path.isfile(file_to_transfer): os.remove(file_to_transfer) @@ -117,24 +121,36 @@ def sftp_merge_and_send(sftp_config: dict, file_path_on_server: str, file_server os.remove(temporary_file) logging.info('Files for ' + file_to_transfer + ' merged') # Then send it to ftp - if not (s.isdir(file_server_location)): - s.mkdir(file_server_location) - with s.cd(file_server_location): - s.put(localpath=file_to_transfer, preserve_mtime=True, - callback=lambda current, total: - SFTPUploader.file_upload_progress(current, total, file_to_transfer, - file_transferred_callback)) - logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...') + if not (SFTPUploader.isdir(client, file_server_location)): + client.mkdir(file_server_location) + + file_name = os.path.basename(file_to_transfer) + remote_file_name = file_server_location + '/' + file_name + local_attr = os.stat(file_to_transfer) + logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...') + client.put(localpath=file_to_transfer, remotepath=remote_file_name, + callback=lambda current, total: + SFTPUploader.file_upload_progress(current, total, file_to_transfer, + file_transferred_callback)) + # Update time on server + times = (local_attr.st_atime, local_attr.st_mtime) + client.utime(remote_file_name, times) + # Move the local file in the transferred directory if os.path.isfile(file_transferred_location): os.remove(file_transferred_location) os.replace(file_to_transfer, file_transferred_location) - return True - except (pysftp.exceptions.ConnectionException, pysftp.CredentialException, - pysftp.AuthenticationException, pysftp.HostKeysException, - paramiko.SSHException, paramiko.PasswordRequiredException) as exc: - logging.error('Error occurred transferring ' + file_path_on_server + ': ' + str(exc)) + + except (BadHostKeyException, SSHException, AuthenticationException, socket.error, IOError) as exc: + err_msg = str(exc) + if hasattr(exc, 'message'): + err_msg = exc.message + ' ' + err_msg + if file_to_transfer: + logging.error('Error occurred transferring ' + str(file_to_transfer) + ': ' + err_msg) + else: + logging.error('Error occured while trying to transfer: ' + err_msg) return False + return True @staticmethod def file_upload_progress(current_bytes: int, total_bytes: int, filename: str = 'Unknown', @@ -144,15 +160,16 @@ def file_upload_progress(current_bytes: int, total_bytes: int, filename: str = ' if pc_transferred >= 100 and file_transferred_callback: file_transferred_callback(filename) - @staticmethod - def sftp_sync(sftp_config: dict, remote_base_path: str, local_base_path: str): - remote_url = sftp_config['username'] + ":" + sftp_config['password'] + '@' + sftp_config["hostname"] + ":" + \ - remote_base_path - - cloner = sftpclone.SFTPClone(local_path=local_base_path, remote_url=remote_url, port=sftp_config["port"], - delete=False, allow_unknown=True) - - cloner.run() + # @staticmethod + # def sftp_sync(sftp_config: dict, remote_base_path: str, local_base_path: str): + # remote_url = sftp_config['username'] + ":" + sftp_config['password'] + '@' + sftp_config["hostname"] + ":" + \ + # remote_base_path + # + # cloner = sftpclone.SFTPClone(local_path=local_base_path, remote_url=remote_url, port=sftp_config["port"], + # delete=False, allow_unknown=True) + # + # cloner.run() + # return @staticmethod def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: str, check_internet: bool = True): @@ -165,7 +182,7 @@ def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: st # Wait for internet connection # PiHubHardware.wait_for_internet_infinite () logging.info("SyncServer: Testing the internet connection...") - while not(Network.is_internet_connected()): + while not (Network.is_internet_connected()): logging.info("SyncServer: Connection failed, retry sync in 10min...") time.sleep(600) logging.info("SyncServer: Pass, syncing files to server...") @@ -187,3 +204,34 @@ def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: st file_to_transfer=filename_2_transfer, check_internet=check_internet) logging.info('file at boot ' + str(filename_2_transfer) + ' synced') os.rmdir(folders[i]) + + @staticmethod + def makedirs(client: SFTPClient, remotedir: str): + if SFTPUploader.isdir(client, remotedir): + pass + + elif SFTPUploader.isfile(client, remotedir): + raise OSError("a file with the same name as the remotedir, " + "'%s', already exists." % remotedir) + else: + head, tail = os.path.split(remotedir) + if head and not SFTPUploader.isdir(client, head): + SFTPUploader.makedirs(client, head) + if tail: + client.mkdir(remotedir) + + @staticmethod + def isdir(client: SFTPClient, remotepath: str) -> bool: + try: + result = stat.S_ISDIR(client.lstat(remotepath).st_mode) + except IOError: # no such file + result = False + return result + + @staticmethod + def isfile(client: SFTPClient, remotepath: str) -> bool: + try: + result = stat.S_ISREG(client.lstat(remotepath).st_mode) + except IOError: # no such file + result = False + return result diff --git a/requirements.txt b/requirements.txt index 63d4d63..2e2911e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -pysftp~=0.2.9 -paramiko~=2.7.2 -sftpclone \ No newline at end of file +fabric == 3.2.2 +paramiko ~= 3.3.1 \ No newline at end of file