Skip to content

Commit

Permalink
Refs #11. Removed sftpclone, sftpclient from requirements, and moved …
Browse files Browse the repository at this point in the history 
…to fabric
  • Loading branch information
@SBriere
SBriere committed Dec 5, 2023
1 parent d80451f commit 341cc62
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 82 deletions.
8 changes: 8 additions & 0 deletions create_conda_venv.bat
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
@echo off
echo Create or update conda venv
call conda install -m -c conda-forge -y --copy -p venv python=3.10
echo Activating venv
call conda activate .\venv
echo Installing requirements
call pip install -r requirements.txt
call conda deactivate
7 changes: 7 additions & 0 deletions create_conda_venv.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
echo "Create or update conda venv"
conda install -m -c conda-forge -y --copy -p $PWD/venv python=3.10
echo "Activating venv"
conda activate $PWD/venv
echo "Installing requirements (make sure git submodules are installed)..."
$PWD/venv/bin/pip install -r $PWD/requirements.txt
conda deactivate
1 change: 1 addition & 0 deletions libs/servers/WatchServer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def sync_files(self, check_internet: bool = True):
self.synching_files = True
# Build list of files to transfer
base_folder = self.data_path + '/ToProcess/'
base_folder = base_folder.replace('/', os.sep)
files = []
full_files = []
file_folders = []
Expand Down
206 changes: 127 additions & 79 deletions libs/uploaders/SFTPUploader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,19 @@
# Authors: Simon Brière, Eng. MASc.
# Mathieu Hamel, Eng. MASc.
##################################################
import paramiko
import pysftp
import logging
import os
import time
import socket
import stat
from os import walk
from libs.hardware.PiHubHardware import PiHubHardware
from os import makedirs

from sftpclone import sftpclone
import fabric
from paramiko import BadHostKeyException, AuthenticationException, SSHException, SFTPClient

from libs.hardware.PiHubHardware import PiHubHardware
from libs.utils.Network import Network
import time
from os import makedirs


class SFTPUploader:
Expand All @@ -28,55 +29,57 @@ def sftp_send(sftp_config: dict, files_directory_on_server: [str], files_to_tran
PiHubHardware.ensure_internet_is_available()

# Do the transfer
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
logging.info('About to send files to server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"]))
file_to_transfer = None
try:
with pysftp.Connection(host=sftp_config["hostname"], username=sftp_config["username"],
password=sftp_config["password"], port=sftp_config["port"],
cnopts=cnopts) as s:
with fabric.Connection(host=sftp_config["hostname"], user=sftp_config["username"],
connect_kwargs={'password': sftp_config["password"]},
port=sftp_config["port"], connect_timeout=10) as c:
c.open()
client: SFTPClient = c.sftp()
for file_server_location, file_to_transfer in zip(files_directory_on_server, files_to_transfer):
if not (s.isdir(file_server_location)):
s.makedirs(file_server_location)
# s.mkdir(file_server_location)
with s.cd(file_server_location):
file_name = os.path.basename(file_to_transfer)
# Query file size from server and compare to local file size
try:
remote_attr = s.stat(remotepath=file_server_location + '/' + file_name)
remote_size = remote_attr.st_size
local_attr = os.stat(file_to_transfer)
local_size = local_attr.st_size
if local_size == remote_size:
# Same size on server as local file, skip!
logging.info('Skipping ' + file_to_transfer + ': already present on server.')
if file_transferred_callback:
file_transferred_callback(file_to_transfer)
continue
except IOError:
# File not on server = ok, continue!
pass
# TODO: Use pysftp.Connection.stat() to find file size and only send if is different
s.put(localpath=file_to_transfer, preserve_mtime=True,
callback=lambda current, total:
SFTPUploader.file_upload_progress(current, total, file_to_transfer,
file_transferred_callback))
logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...')
# s.close()
# logging.info('File ' + file_to_transfer + ' sent.')
except (pysftp.exceptions.ConnectionException, pysftp.CredentialException,
pysftp.AuthenticationException, pysftp.HostKeysException,
paramiko.SSHException, paramiko.PasswordRequiredException) as exc:
# Create directories on server if needed
if not (SFTPUploader.isdir(client, file_server_location)):
SFTPUploader.makedirs(client, file_server_location)

# Move to directory
# client.chdir(file_server_location)
file_name = os.path.basename(file_to_transfer)
# Query file size from server and compare to local file size
local_attr = os.stat(file_to_transfer)
local_size = local_attr.st_size
remote_file_name = file_server_location + '/' + file_name
try:
remote_attr = client.lstat(remote_file_name)
remote_size = remote_attr.st_size
if local_size == remote_size:
# Same size on server as local file, skip!
logging.info('Skipping ' + file_to_transfer + ': already present on server.')
if file_transferred_callback:
file_transferred_callback(file_to_transfer)
continue
except IOError:
# File not on server = ok, continue!
pass

logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...')
client.put(localpath=file_to_transfer, remotepath=remote_file_name, confirm=True,
callback=lambda current, total:
SFTPUploader.file_upload_progress(current, total, file_to_transfer,
file_transferred_callback))
# Update time on server
times = (local_attr.st_atime, local_attr.st_mtime)
client.utime(remote_file_name, times)

except (BadHostKeyException, SSHException, AuthenticationException, socket.error, IOError) as exc:
err_msg = str(exc)
if exc.message:
if hasattr(exc, 'message'):
err_msg = exc.message + ' ' + err_msg
if file_to_transfer:
logging.error('Error occurred transferring ' + file_to_transfer + ': ' + err_msg)
logging.error('Error occurred transferring ' + str(file_to_transfer) + ': ' + err_msg)
else:
logging.error('Error occured while trying to transfer: ' + err_msg)
return False

logging.info('Files transfer complete!')
return True

Expand All @@ -88,25 +91,26 @@ def sftp_merge_and_send(sftp_config: dict, file_path_on_server: str, file_server
# Check if Internet connected
if check_internet:
PiHubHardware.ensure_internet_is_available()
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
logging.info('About to get file from server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"]))

logging.info('About to get files from server at ' + sftp_config["hostname"] + ':' + str(sftp_config["port"]))
try:
with pysftp.Connection(host=sftp_config["hostname"], username=sftp_config["username"],
password=sftp_config["password"], port=sftp_config["port"],
cnopts=cnopts) as s:
with fabric.Connection(host=sftp_config["hostname"], user=sftp_config["username"],
connect_kwargs={'password': sftp_config["password"]},
port=sftp_config["port"], connect_timeout=10) as c:
c.open()
client: SFTPClient = c.sftp()
# Check if the file exist on remote and get it locally
if not (s.isfile(file_path_on_server)):
if not (SFTPUploader.isfile(client, file_path_on_server)):
logging.info('No file on server to merge with ' + file_to_transfer)
else:
s.get(file_path_on_server, temporary_file)
client.get(file_path_on_server, temporary_file)
# Now do the merge in the local file
m_point = "/mnt/app"
lines = lambda f: [l for l in open(f).read().splitlines()]
lines = lambda f: [line for line in open(f).read().splitlines()]
lines1 = lines(temporary_file)
lines2 = lines(file_to_transfer)
checks = [l.split(m_point)[-1] for l in lines1]
for item in sum([[l for l in lines2 if c in l] for c in checks], []):
checks = [line.split(m_point)[-1] for line in lines1]
for item in sum([[line for line in lines2 if c in line] for c in checks], []):
lines2.remove(item)
if os.path.isfile(file_to_transfer):
os.remove(file_to_transfer)
Expand All @@ -117,24 +121,36 @@ def sftp_merge_and_send(sftp_config: dict, file_path_on_server: str, file_server
os.remove(temporary_file)
logging.info('Files for ' + file_to_transfer + ' merged')
# Then send it to ftp
if not (s.isdir(file_server_location)):
s.mkdir(file_server_location)
with s.cd(file_server_location):
s.put(localpath=file_to_transfer, preserve_mtime=True,
callback=lambda current, total:
SFTPUploader.file_upload_progress(current, total, file_to_transfer,
file_transferred_callback))
logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...')
if not (SFTPUploader.isdir(client, file_server_location)):
client.mkdir(file_server_location)

file_name = os.path.basename(file_to_transfer)
remote_file_name = file_server_location + '/' + file_name
local_attr = os.stat(file_to_transfer)
logging.info('Sending ' + file_to_transfer + ' to ' + file_server_location + ' ...')
client.put(localpath=file_to_transfer, remotepath=remote_file_name,
callback=lambda current, total:
SFTPUploader.file_upload_progress(current, total, file_to_transfer,
file_transferred_callback))
# Update time on server
times = (local_attr.st_atime, local_attr.st_mtime)
client.utime(remote_file_name, times)

# Move the local file in the transferred directory
if os.path.isfile(file_transferred_location):
os.remove(file_transferred_location)
os.replace(file_to_transfer, file_transferred_location)
return True
except (pysftp.exceptions.ConnectionException, pysftp.CredentialException,
pysftp.AuthenticationException, pysftp.HostKeysException,
paramiko.SSHException, paramiko.PasswordRequiredException) as exc:
logging.error('Error occurred transferring ' + file_path_on_server + ': ' + str(exc))

except (BadHostKeyException, SSHException, AuthenticationException, socket.error, IOError) as exc:
err_msg = str(exc)
if hasattr(exc, 'message'):
err_msg = exc.message + ' ' + err_msg
if file_to_transfer:
logging.error('Error occurred transferring ' + str(file_to_transfer) + ': ' + err_msg)
else:
logging.error('Error occured while trying to transfer: ' + err_msg)
return False
return True

@staticmethod
def file_upload_progress(current_bytes: int, total_bytes: int, filename: str = 'Unknown',
Expand All @@ -144,15 +160,16 @@ def file_upload_progress(current_bytes: int, total_bytes: int, filename: str = '
if pc_transferred >= 100 and file_transferred_callback:
file_transferred_callback(filename)

@staticmethod
def sftp_sync(sftp_config: dict, remote_base_path: str, local_base_path: str):
remote_url = sftp_config['username'] + ":" + sftp_config['password'] + '@' + sftp_config["hostname"] + ":" + \
remote_base_path

cloner = sftpclone.SFTPClone(local_path=local_base_path, remote_url=remote_url, port=sftp_config["port"],
delete=False, allow_unknown=True)

cloner.run()
# @staticmethod
# def sftp_sync(sftp_config: dict, remote_base_path: str, local_base_path: str):
# remote_url = sftp_config['username'] + ":" + sftp_config['password'] + '@' + sftp_config["hostname"] + ":" + \
# remote_base_path
#
# cloner = sftpclone.SFTPClone(local_path=local_base_path, remote_url=remote_url, port=sftp_config["port"],
# delete=False, allow_unknown=True)
#
# cloner.run()
# return

@staticmethod
def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: str, check_internet: bool = True):
Expand All @@ -165,7 +182,7 @@ def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: st
# Wait for internet connection
# PiHubHardware.wait_for_internet_infinite ()
logging.info("SyncServer: Testing the internet connection...")
while not(Network.is_internet_connected()):
while not (Network.is_internet_connected()):
logging.info("SyncServer: Connection failed, retry sync in 10min...")
time.sleep(600)
logging.info("SyncServer: Pass, syncing files to server...")
Expand All @@ -187,3 +204,34 @@ def sftp_sync_last(sftp_config: dict, remote_base_path: str, local_base_path: st
file_to_transfer=filename_2_transfer, check_internet=check_internet)
logging.info('file at boot ' + str(filename_2_transfer) + ' synced')
os.rmdir(folders[i])

@staticmethod
def makedirs(client: SFTPClient, remotedir: str):
if SFTPUploader.isdir(client, remotedir):
pass

elif SFTPUploader.isfile(client, remotedir):
raise OSError("a file with the same name as the remotedir, "
"'%s', already exists." % remotedir)
else:
head, tail = os.path.split(remotedir)
if head and not SFTPUploader.isdir(client, head):
SFTPUploader.makedirs(client, head)
if tail:
client.mkdir(remotedir)

@staticmethod
def isdir(client: SFTPClient, remotepath: str) -> bool:
try:
result = stat.S_ISDIR(client.lstat(remotepath).st_mode)
except IOError: # no such file
result = False
return result

@staticmethod
def isfile(client: SFTPClient, remotepath: str) -> bool:
try:
result = stat.S_ISREG(client.lstat(remotepath).st_mode)
except IOError: # no such file
result = False
return result
5 changes: 2 additions & 3 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
pysftp~=0.2.9
paramiko~=2.7.2
sftpclone
fabric == 3.2.2
paramiko ~= 3.3.1

0 comments on commit 341cc62

Please sign in to comment.