diff --git a/lib/oc-tests/new_locking/test_downloadFilePlusAction.py b/lib/oc-tests/new_locking/test_downloadFilePlusAction.py new file mode 100644 index 0000000..8f45e5e --- /dev/null +++ b/lib/oc-tests/new_locking/test_downloadFilePlusAction.py @@ -0,0 +1,200 @@ +__doc__ = """ +Test locking feature. The test will download a file big enough and perform an operation +while the download is running. Each test set will run a different operation. + +Test sets: +* download + overwrite +* download + delete +* download + info (propfind) +* download + rename +* download + move +* download + delete parent folder +* download + rename folder +* download + move folder + ++-------------+---------------------------+----------------+ +| step number | downloader | doer | ++-------------+---------------------------+----------------+ +| 2 | upload big file | | ++-------------+---------------------------+----------------+ +| 3 | download big file (async) | | ++-------------+---------------------------+----------------+ +| 4 | | perform action | ++-------------+---------------------------+----------------+ +| 5 | check result and cleanup | check result | ++-------------+---------------------------+----------------+ + +""" +import time +import os +import re +import tempfile +import owncloud +import smashbox.utilities.pyocclient_wrapper +from smashbox.utilities import * +from smashbox.script import config as sconf + +def parse_worker_number(worker_name): + match = re.search(r'(\d+)$', worker_name) + if match is not None: + return int(match.group()) + else: + return None + +testsets = [ + { 'action_method': 'put_file_contents', + 'action_args': ('/folder/bigfile.dat', '123'*50), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_filesize', + 'extra_check_params': ('/folder/bigfile.dat', 3*50) + }, + { 'action_method': 'delete', + 'action_args': ('/folder/bigfile.dat',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_file_not_exists', + 'extra_check_params': ('/folder/bigfile.dat',) + }, + { 'action_method': 'file_info', + 'action_args': ('/folder/bigfile.dat',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': None, + 'extra_check_params': () + }, + { 'action_method': 'move', + 'action_args': ('/folder/bigfile.dat', '/folder/bigrenamed.dat'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_exists_second_not', + 'extra_check_params': ('/folder/bigrenamed.dat', '/folder/bigfile.dat') + }, + { 'action_method': 'move', + 'action_args': ('/folder/bigfile.dat', '/folder2/bigfile.dat'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_exists_second_not', + 'extra_check_params': ('/folder2/bigfile.dat', '/folder/bigfile.dat') + }, + { 'action_method': 'delete', + 'action_args': ('/folder',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_all_files_not_exists', + 'extra_check_params': ('/folder/bigfile.dat', '/folder') + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder-renamed'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder-renamed', '/folder-renamed/bigfile.dat'), + ('/folder', '/folder/bigfile.dat')) + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder2/folder'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder2', '/folder2/folder', '/folder2/folder/bigfile.dat'), + ('/folder', '/folder/bigfile.dat')) + }, +] + +@add_worker +def setup(step): + step(1, 'setup') + reset_owncloud_account(num_test_users=config.get('accounts', 1)) + reset_rundir() + +def downloader(step): + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'create big file') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + d = make_workdir() + + list_files(d) + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + + # sync a big file + target_filename = os.path.join(d, 'folder', 'bigfile.dat') + mkdir(os.path.join(d, 'folder')) + mkdir(os.path.join(d, 'folder2')) + createfile(target_filename,'10',count=1000,bs=10000) + sum5 = md5sum(target_filename) + + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + list_files(d) + + step(3, 'download file async') + + try: + tmpfile = tempfile.mkstemp() + # download the file asynchronously + download_thread = client_wrapper.do_action_async('get_file', '/folder/bigfile.dat', tmpfile[1]) + + step(5, 'check result and cleanup') + + # wait until the download finish + download_thread[0].join() + download_result = download_thread[1].get() + if isinstance(download_result, Exception): + raise download_result + else: + error_check(download_result, 'download file failed') + + sum5_2 = md5sum(tmpfile[1]) + # check both md5 matches + logger.debug('checking md5sum of the downloaded files') + error_check(sum5 == sum5_2, 'uploaded file is different than the downloaded file [%s] - [%s]' % (sum5, sum5_2)) + finally: + # remove temporal file + os.remove(tmpfile[1]) + +def doer(step): + method = config.get('action_method', 'put_file_contents') + args = config.get('action_args', ('/folder/bigfile.dat', '123'*50)) + kwargs = config.get('action_kwargs', {}) + + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'synced setup') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + step(4, 'action over file') + + retry_action = False + # perform the action + try: + result = client_wrapper.do_action(method, *args, **kwargs) + except owncloud.ResponseError as e: + logger.debug('%s action failed. Checking the status to know if the file is locked' % (method,)) + error_check(e.status_code == 423, 'unexpected status code [%i] : %s' % (e.status_code, e.get_resource_body())) + retry_action = True + + step(6, 'check results') + if retry_action: + result = client_wrapper.do_action(method, *args, **kwargs) + + # check successful result + logger.debug('check %s method finished correctly' % method) + error_check(result, method + ' action didn\'t finish correctly') + + # perform extra check + check = config.get('extra_check', None) + if check: + logger.debug('additional check %s' % check) + check_params = config.get('extra_check_params', ()) + error_check(getattr(client_wrapper, check)(*check_params), 'extra check failed: %s %s' % (check, check_params)) + +# add workers +for i in range(1, config.get('accounts', 1) + 1): + add_worker(downloader, name='downloader_%s' % (i,)) + add_worker(doer, name='doer_%s' % (i,)) diff --git a/lib/oc-tests/new_locking/test_overwriteFilePlusAction.py b/lib/oc-tests/new_locking/test_overwriteFilePlusAction.py new file mode 100644 index 0000000..119f68d --- /dev/null +++ b/lib/oc-tests/new_locking/test_overwriteFilePlusAction.py @@ -0,0 +1,274 @@ +__doc__ = """ +Test locking feature. The test will ovewrite a file big enough and perform an operation +while the overwrite is running. Each test set will run a different operation. +Chunked uploads are currently outside of the tests + +Test sets: +* overwrite + overwrite +* overwrite + download file +* overwrite + info (propfind) +* overwrite + download folder as zip +* overwrite + delete file +* overwrite + delete folder +* overwrite + rename file +* overwrite + move file +* overwrite + rename folder +* overwrite + move folder + ++-------------+----------------------------+--------------------+ +| step number | overwriter | doer | ++-------------+----------------------------+--------------------+ +| 2 | upload big file | create working dir | ++-------------+----------------------------+--------------------+ +| 3 | overwrite big file (async) | | ++-------------+----------------------------+--------------------+ +| 4 | | perform action | ++-------------+----------------------------+--------------------+ +| 5 | check result and cleanup | | ++-------------+----------------------------+--------------------+ +| 6 | | check result | ++-------------+----------------------------+--------------------+ +""" + +import os +import re +import tempfile +import uuid +import owncloud +import zipfile +import smashbox.utilities.pyocclient_wrapper +from smashbox.utilities import * +from smashbox.script import config as sconf + +def check_local_filesize(localpath, size): + '''username and password remains to keep the expected signature''' + return os.path.getsize(localpath) == size + +def check_zip_contents(localpath, content_list): + checked_result = False + with zipfile.ZipFile(localpath, 'r') as myzip: + checked_result = myzip.namelist() == content_list + return checked_result + +def parse_worker_number(worker_name): + match = re.search(r'(\d+)$', worker_name) + if match is not None: + return int(match.group()) + else: + return None + + +testsets = [ + { 'action_method': 'put_file_contents', + 'action_args': ('/folder/bigfile.dat', '123'*50), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_filesize', + 'extra_check_params': ('/folder/bigfile.dat', 3*50), + 'overwrite_kwargs' : {'chunked': False}, + }, +# chunked uploads aren't supported for the moment +# { 'action_method': 'put_file_contents', +# 'action_args': ('/folder/bigfile.dat', '123'*50), +# 'action_kwargs': {'pyocactiondebug' : True}, +# 'accounts': sconf.oc_number_test_users, +# 'extra_check': 'check_filesize', +# 'extra_check_params': ('/folder/bigfile.dat', 3*50), +# 'overwrite_kwargs' : {'chunked' : True, 'chunk_size' : 1024*1024}, #1MB +# }, + { 'action_method': 'get_file', + 'action_args': ['/folder/bigfile.dat', 'bigfile.dat'], + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_local_filesize', + 'extra_check_params': ['bigfile.dat', 10000*1000], + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'file_info', + 'action_args': ('/folder/bigfile.dat',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': None, + 'extra_check_params': (), + 'overwrite_kwargs' : {'chunked': False}, + }, +# currently failing due to https://github.com/owncloud/core/issues/16960 +# { 'action_method': 'get_directory_as_zip', +# 'action_args': ['/folder', 'folder.zip'], +# 'action_kwargs': {}, +# 'accounts': sconf.oc_number_test_users, +# 'extra_check': 'check_zip_contents', +# 'extra_check_params': ['folder.zip', ['folder/', 'folder/bigfile.dat']], +# 'overwrite_kwargs' : {'chunked': False}, +# }, + { 'action_method': 'delete', + 'action_args': ('/folder/bigfile.dat',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_file_not_exists', + 'extra_check_params': ('/folder/bigfile.dat',), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'delete', + 'action_args': ('/folder',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_all_files_not_exists', + 'extra_check_params': ('/folder/bigfile.dat', '/folder'), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder/bigfile.dat', '/folder/bigrenamed.dat'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_exists_second_not', + 'extra_check_params': ('/folder/bigrenamed.dat', '/folder/bigfile.dat'), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder/bigfile.dat', '/folder2/bigfile.dat'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_exists_second_not', + 'extra_check_params': ('/folder2/bigfile.dat', '/folder/bigfile.dat'), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder-renamed'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder-renamed', '/folder-renamed/bigfile.dat'), + ('/folder', '/folder/bigfile.dat')), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder2/folder'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder2', '/folder2/folder', '/folder2/folder/bigfile.dat'), + ('/folder', '/folder/bigfile.dat')), + 'overwrite_kwargs' : {'chunked': False}, + }, +] + +@add_worker +def setup(step): + step(1, 'setup') + reset_owncloud_account(num_test_users=config.get('accounts', 1)) + reset_rundir() + +def overwriter(step): + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'create big file') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + d = make_workdir() + + list_files(d) + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + + # sync a big file + target_filename = os.path.join(d, 'folder', 'bigfile.dat') + mkdir(os.path.join(d, 'folder')) + mkdir(os.path.join(d, 'folder2')) + createfile(target_filename,'10',count=1000,bs=10000) + sum_orig = md5sum(target_filename) + + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + list_files(d) + + + step(3, 'overwrite file') + + try: + tmpfile = tempfile.mkstemp() + createfile(tmpfile[1], '5', count=1000, bs=10000) + sum_new = md5sum(tmpfile[1]) + + overwrite_kwargs = config.get('overwrite_kwargs', {}) + overwrite_thread = client_wrapper.do_action_async('put_file', '/folder/bigfile.dat', tmpfile[1], **overwrite_kwargs) + + step(5, 'check result and cleanup') + + # wait until the overwrite finish + overwrite_thread[0].join() + overwrite_result = overwrite_thread[1].get() + if isinstance(overwrite_result, Exception): + raise overwrite_result + else: + error_check(overwrite_result, 'put file failed') + + # download the file to check that it has been overwritten + tmpfile2 = tempfile.mkstemp() + get_file_result = client_wrapper.do_action('get_file', '/folder/bigfile.dat', tmpfile2[1]) + + sum_downloaded = md5sum(tmpfile2[1]) + + # check both md5 matches + error_check(get_file_result, 'overwritten file failed to download') + logger.debug('checking md5sum of the downloaded files') + error_check(sum_orig != sum_new, 'original file didn\'t get overwritten') + error_check(sum_new == sum_downloaded, 'overwritten file is different than the downloaded file [%s] - [%s]' % (sum_new, sum_downloaded)) + finally: + # remove temporal files + for tfile in ('tmpfile', 'tmpfile2'): + if tfile in locals(): + os.remove(locals()[tfile][1]) + +def doer(step): + method = config.get('action_method', 'put_file_contents') + args = config.get('action_args', ('/folder/bigfile.dat', '123'*50)) + kwargs = config.get('action_kwargs', {}) + + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'create working dir in case it\'s needed') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + d = make_workdir() + + if method in ('get_file', 'get_directory_as_zip'): + # we need to cheat at this two method to make them work properly + args[1] = os.path.join(d, args[1]) + + step(4, 'action over file') + + retry_action = False + # perform the action + try: + result = client_wrapper.do_action(method, *args, **kwargs) + except owncloud.ResponseError as e: + logger.debug('%s action failed. Checking the status to know if the file is locked' % (method,)) + error_check(e.status_code == 423, 'unexpected status code [%i] : %s' % (e.status_code, e.get_resource_body())) + retry_action = True + + step(6, 'check results') + + if retry_action: + result = client_wrapper.do_action(method, *args, **kwargs) + # check successful result + error_check(result, method + ' action didn\'t finish correctly') + + # perform extra check + check = config.get('extra_check', None) + if check: + logger.debug('additional check %s' % check) + check_params = config.get('extra_check_params', ()) + if method in ('get_file', 'get_directory_as_zip'): + # we need to cheat at this two method to make them work properly + check_params[0] = os.path.join(d, check_params[0]) + error_check(globals()[check](*check_params), 'extra check failed: %s %s' % (check, check_params)) + else: + error_check(getattr(client_wrapper, check)(*check_params), 'extra check failed: %s %s' % (check, check_params)) + +# add workers +for i in range(1, config.get('accounts', 1) + 1): + add_worker(overwriter, name='overwriter_%s' % (i,)) + add_worker(doer, name='doer_%s' % (i,)) diff --git a/lib/oc-tests/new_locking/test_uploadFilePlusAction.py b/lib/oc-tests/new_locking/test_uploadFilePlusAction.py new file mode 100644 index 0000000..addc2f7 --- /dev/null +++ b/lib/oc-tests/new_locking/test_uploadFilePlusAction.py @@ -0,0 +1,222 @@ +__doc__ = """ +Test locking feature. The test will upload a file big enough and perform an operation +while the upload is running. Each test set will run a different operation. +Chunked uploads are currently outside of the tests + +Test sets: +* upload + upload (same file) +* upload + delete folder +* upload + rename folder +* upload + move folder + ++-------------+----------------------------+--------------------+ +| step number | uploader | doer | ++-------------+----------------------------+--------------------+ +| 2 | upload big file | create working dir | ++-------------+----------------------------+--------------------+ +| 3 | overwrite big file (async) | | ++-------------+----------------------------+--------------------+ +| 4 | | perform action | ++-------------+----------------------------+--------------------+ +| 5 | check result and cleanup | | ++-------------+----------------------------+--------------------+ +| 6 | | check result | ++-------------+----------------------------+--------------------+ +""" + +import os +import re +import tempfile +import uuid +import owncloud +import zipfile +import smashbox.utilities.pyocclient_wrapper +from smashbox.utilities import * +from smashbox.script import config as sconf + +def check_zip_contents(localpath, content_list): + checked_result = False + with zipfile.ZipFile(localpath, 'r') as myzip: + checked_result = myzip.namelist() == content_list + return checked_result + +def parse_worker_number(worker_name): + match = re.search(r'(\d+)$', worker_name) + if match is not None: + return int(match.group()) + else: + return None + + +testsets = [ + { 'action_method': 'put_file_contents', + 'action_args': ('/folder/new_bigfile.dat', '123'*50), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_filesize', + 'extra_check_params': ('/folder/new_bigfile.dat', 3*50), + 'overwrite_kwargs' : {'chunked': False}, + }, +# chunked uploads aren't supported for the moment +# { 'action_method': 'put_file_contents', +# 'action_args': ('/folder/bigfile.dat', '123'*50), +# 'action_kwargs': {'pyocactiondebug' : True}, +# 'accounts': sconf.oc_number_test_users, +# 'extra_check': 'check_filesize', +# 'extra_check_params': ('/folder/bigfile.dat', 3*50), +# 'overwrite_kwargs' : {'chunked' : True, 'chunk_size' : 1024*1024}, #1MB +# }, +# currently failing due to https://github.com/owncloud/core/issues/16960 +# { 'action_method': 'get_directory_as_zip', +# 'action_args': ['/folder', 'folder.zip'], +# 'action_kwargs': {}, +# 'accounts': sconf.oc_number_test_users, +# 'extra_check': 'check_zip_contents', +# 'extra_check_params': ['folder.zip', ['folder/', 'folder/bigfile.dat', 'folder/new_bigfile.dat']], +# 'overwrite_kwargs' : {'chunked': False}, +# }, + { 'action_method': 'delete', + 'action_args': ('/folder',), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_all_files_not_exists', + 'extra_check_params': ('/folder/new_bigfile.dat', '/folder/bigfile.dat', '/folder'), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder-renamed'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder-renamed', '/folder-renamed/bigfile.dat', '/folder-renamed/new_bigfile.dat'), + ('/folder', '/folder/bigfile.dat', '/folder/new_bigfile.dat')), + 'overwrite_kwargs' : {'chunked': False}, + }, + { 'action_method': 'move', + 'action_args': ('/folder', '/folder2/folder'), + 'action_kwargs': {}, + 'accounts': sconf.oc_number_test_users, + 'extra_check': 'check_first_list_exists_second_list_not', + 'extra_check_params': (('/folder2', '/folder2/folder', '/folder2/folder/bigfile.dat', '/folder2/folder/new_bigfile.dat'), + ('/folder', '/folder/bigfile.dat', '/folder/new_bigfile.dat')), + 'overwrite_kwargs' : {'chunked': False}, + }, +] + +@add_worker +def setup(step): + step(1, 'setup') + reset_owncloud_account(num_test_users=config.get('accounts', 1)) + reset_rundir() + +def uploader(step): + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'create big file') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + d = make_workdir() + + list_files(d) + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + + # sync a big file + target_filename = os.path.join(d, 'folder', 'bigfile.dat') + mkdir(os.path.join(d, 'folder')) + mkdir(os.path.join(d, 'folder2')) + createfile(target_filename,'10',count=1000,bs=10000) + + run_ocsync(d, user_num=None if process_number <= 0 else process_number) + list_files(d) + + + step(3, 'upload new file') + + try: + tmpfile = tempfile.mkstemp() + createfile(tmpfile[1], '5', count=1000, bs=10000) + sum_new = md5sum(tmpfile[1]) + + upload_kwargs = config.get('overwrite_kwargs', {}) + upload_thread = client_wrapper.do_action_async('put_file', '/folder/new_bigfile.dat', tmpfile[1], **upload_kwargs) + + step(5, 'check result and cleanup') + + # wait until the upload finish + upload_thread[0].join() + upload_result = upload_thread[1].get() + if isinstance(upload_result, Exception): + raise upload_result + else: + error_check(upload_result, 'put file failed') + + # download the file to check that it has been overwritten + tmpfile2 = tempfile.mkstemp() + get_file_result = client_wrapper.do_action('get_file', '/folder/new_bigfile.dat', tmpfile2[1]) + + sum_downloaded = md5sum(tmpfile2[1]) + + # check both md5 matches + error_check(get_file_result, 'uploaded file failed to download') + logger.debug('checking md5sum of the downloaded files') + error_check(sum_new == sum_downloaded, 'overwritten file is different than the downloaded file [%s] - [%s]' % (sum_new, sum_downloaded)) + finally: + # remove temporal files + for tfile in ('tmpfile', 'tmpfile2'): + if tfile in locals(): + os.remove(locals()[tfile][1]) + +def doer(step): + method = config.get('action_method', 'put_file_contents') + args = config.get('action_args', ('/folder/new_bigfile.dat', '123'*50)) + kwargs = config.get('action_kwargs', {}) + + process_number = parse_worker_number(reflection.getProcessName()) + user_account = sconf.oc_account_name if process_number <= 0 else '%s%i' % (sconf.oc_account_name, process_number) + + step(2, 'create working dir in case it\'s needed') + + client_wrapper = pyocclient_wrapper.pyocclient_wrapper(pyocclient_wrapper.pyocclient_basic_url(), user_account, sconf.oc_account_password, debug=True) + + d = make_workdir() + + if method in ('get_file', 'get_directory_as_zip'): + # we need to cheat at this two method to make them work properly + args[1] = os.path.join(d, args[1]) + + step(4, 'action over file') + + retry_action = False + # perform the action + try: + result = client_wrapper.do_action(method, *args, **kwargs) + except owncloud.ResponseError as e: + logger.debug('%s action failed. Checking the status to know if the file is locked' % (method,)) + error_check(e.status_code == 423, 'unexpected status code [%i] : %s' % (e.status_code, e.get_resource_body())) + retry_action = True + + step(6, 'check results') + + if retry_action: + result = client_wrapper.do_action(method, *args, **kwargs) + # check successful result + error_check(result, method + ' action didn\'t finish correctly') + + # perform extra check + check = config.get('extra_check', None) + if check: + logger.debug('additional check %s' % check) + check_params = config.get('extra_check_params', ()) + if method in ('get_file', 'get_directory_as_zip'): + # we need to cheat at this two method to make them work properly + check_params[0] = os.path.join(d, check_params[0]) + error_check(globals()[check](*check_params), 'extra check failed: %s %s' % (check, check_params)) + else: + error_check(getattr(client_wrapper, check)(*check_params), 'extra check failed: %s %s' % (check, check_params)) + +# add workers +for i in range(1, config.get('accounts', 1) + 1): + add_worker(uploader, name='uploader_%s' % (i,)) + add_worker(doer, name='doer_%s' % (i,)) diff --git a/python/smashbox/utilities/__init__.py b/python/smashbox/utilities/__init__.py index ae96d9a..5f0b0b5 100644 --- a/python/smashbox/utilities/__init__.py +++ b/python/smashbox/utilities/__init__.py @@ -10,7 +10,7 @@ def OWNCLOUD_CHUNK_SIZE(factor=1): """Calculate file size as a fraction of owncloud client's default chunk size. """ - return int(20*1024*1024*factor) # 20MB as of client 1.7 + return int(20*1024*1024*factor) # 20MB as of client 1.7 ######## TEST SETUP AND PREPARATION @@ -47,12 +47,12 @@ def finalize_test(): ######### HELPERS def reset_owncloud_account(reset_procedure=None, num_test_users=None): - """ + """ Prepare the test account on the owncloud server (remote state). Run this once at the beginning of the test. The reset_procedure defines what actually happens. If not set then the config default oc_account_reset_procedure applies. - + Normally the account is deleted and recreated ('delete') If reset_procedure is set to 'keep' than the account is not deleted, so the state from the previous run is kept. @@ -83,7 +83,7 @@ def reset_owncloud_account(reset_procedure=None, num_test_users=None): if reset_procedure == 'webdav_delete': webdav_delete('/') # delete the complete webdav endpoint associated with the remote account - webdav_delete('/') # FIXME: workaround current bug in EOS (https://savannah.cern.ch/bugs/index.php?104661) + webdav_delete('/') # FIXME: workaround current bug in EOS (https://savannah.cern.ch/bugs/index.php?104661) # if create if does not exist (for keep or webdav_delete options) webdav_mkcol('/') @@ -92,7 +92,7 @@ def reset_owncloud_account(reset_procedure=None, num_test_users=None): def reset_rundir(reset_procedure=None): """ Prepare the run directory for the current test (local state). Run this once at the beginning of the test. - The reset_procedure defines what actually happens. If not set then the config default rundir_reset_procedure + The reset_procedure defines what actually happens. If not set then the config default rundir_reset_procedure applies. Normally the run directory is deleted ('delete'). To keep the local run directory intact specify "keep". @@ -113,8 +113,8 @@ def reset_rundir(reset_procedure=None): def make_workdir(name=None): - """ Create a worker directory in the current run directory for the test (by default the name is derived from - the worker's name). + """ Create a worker directory in the current run directory for the test (by default the name is derived from + the worker's name). """ from smashbox.utilities import reflection @@ -475,12 +475,12 @@ def hexdump(fn): def list_versions_on_server(fn): - cmd = "%(oc_server_shell_cmd)s md5sum %(oc_server_datadirectory)s/%(oc_account_name)s/files_versions/%(filename)s.v*" % config._dict(filename=os.path.join(config.oc_server_folder, os.path.basename(fn))) # PENDING: bash -x + cmd = "%(oc_server_shell_cmd)s md5sum %(oc_server_datadirectory)s/%(oc_account_name)s/files_versions/%(filename)s.v*" % config._dict(filename=os.path.join(config.oc_server_folder, os.path.basename(fn))) # PENDING: bash -x runcmd(cmd) def hexdump_versions_on_server(fn): - cmd = "%(oc_server_shell_cmd)s hexdump %(oc_server_datadirectory)s/%(oc_account_name)s/files_versions/%(filename)s.v*" % config._dict(filename=os.path.join(config.oc_server_folder, os.path.basename(fn))) # PENDING: bash -x + cmd = "%(oc_server_shell_cmd)s hexdump %(oc_server_datadirectory)s/%(oc_account_name)s/files_versions/%(filename)s.v*" % config._dict(filename=os.path.join(config.oc_server_folder, os.path.basename(fn))) # PENDING: bash -x runcmd(cmd) @@ -517,7 +517,7 @@ def error_check(expr,message=""): """ Assert expr is True. If not, then mark the test as failed but carry on the execution. """ - if not expr: + if not expr: import inspect f=inspect.getouterframes(inspect.currentframe())[1] message=" ".join([message, "%s failed in %s() [\"%s\" at line %s]" %(''.join(f[4]).strip(),f[3],f[1],f[2])]) diff --git a/python/smashbox/utilities/pyocclient_wrapper.py b/python/smashbox/utilities/pyocclient_wrapper.py new file mode 100644 index 0000000..261aefb --- /dev/null +++ b/python/smashbox/utilities/pyocclient_wrapper.py @@ -0,0 +1,227 @@ +import owncloud +import threading +import Queue +from smashbox.script import config + +class pyocclient_wrapper(object): + """ + Simple wrapper over pyocclient : Client class + + If username and password are passed in the constructor, the client will try to login + automatically, otherwise you'll need to login later + + There are 2 basic methods: "do_action" and "do_action_async". Both methods do calls a + client method of your choice with the specific parameters. + The do_action method will execute the action and return the result (no exception is + catched, so exceptions might be thrown depending on the method called) + The do_action_async method will execute the action asynchronously (creating a new thread + and starting it) + + There are additional methods to help with the tests + """ + + def __init__(self, url, username=None, password=None, **kwargs): + """ + Create a new instance setting the url for the client to connect. Any additional + parameter you want to pass to the client can be pass through the kwargs param. + + If you don't pass the username and password, you'll need to make a login request + at some point + + :param url: the url where the pyocclient will connect + :param username: the username to login + :param password: the password to login + """ + client = owncloud.Client(url, **kwargs) + if username != None and password != None: + client.login(username, password) + self._client = client + + def do_action(self, method, *args, **kwargs): + """ + Execute the "method" with the appropiate parameters and return the result. + Exceptions might be thrown depending on the called method. + + :param method: method to be called in the wrapped pyocclient, as a string. The + method should exists + :param args: arguments to be passed to the method + :param kwargs: arguments to be passed to the method + :returns: whatever the called method returns + """ + caller = getattr(self._client, method) + return caller(*args, **kwargs) + + def do_action_async(self, method, *args, **kwargs): + """ + Execute the "method" with the appropiate parameters in an async way. A thread will + be spawned to do the job. + + The method returns a tuple with: + * A thread, to control the execution (mainly to know when the thread finishes) + * A queue, to read the result when the thread finishes) + + The queue to read the result is an instance of Queue.Queue. It can contains the + method's result or the exception thrown by the method (if it was raised). It's + recommended to check the type of the result before making any assumption about + the success or failure of the called method + + :param method: method to be called in the wrapper pyocclient + :param args: arguments to be passed to the method + :param kwargs: arguments to be passed to the method + :returns: tuple with a thread and a queue objects + """ + def caller_wrapper(method, q, *args, **kwargs): + try: + q.put(method(*args, **kwargs)) + except Exception as e: + q.put(e) + + caller = getattr(self._client, method) + result_queue = Queue.Queue() + caller_wrapper_args = (caller, result_queue) + args + thread = threading.Thread(target=caller_wrapper, args=caller_wrapper_args, kwargs=kwargs) + thread.start() + return (thread, result_queue) + + def check_filesize(self, path, size): + """ + Check the size of the file and return True if matches. The size is checked with + whatever the profind call returns. + + If the file isn't found (404 error code) it will return False. For other exceptions, + they will be rethrown + + :param path: the remote path fo the file + :param size: the size we want to check + :returns: True if the size matches, False otherwise + """ + try: + info = self.do_action('file_info', path) + return False if info is None else size == info.get_size() + except owncloud.ResponseError as e: + if e.status_code == 404: + return False + else: + raise e + + def check_file_exists(self, path): + """ + Check if the file exists (based on the 'file_info' call). Returns True if we get + some info, False otherwise (including 404 error). Exceptions other than the 404 + will be rethrown + + :param path: the path to check + :returns: True if the file exists, false otherwise + """ + try: + info = self.do_action('file_info', path) + return False if info is None else True + except owncloud.ResponseError as e: + if e.status_code == 404: + return False + else: + raise e + + def check_file_not_exists(self, path): + """ + Check the file doesn't exist. Same as "not check_file_exists", just for convenience. + + :param path: the path to check + :returns: True if the file doesn't exists, False otherwise + """ + return not self.check_file_exists(path) + + def check_first_exists_second_not(self, path1, path2): + """ + Check that path1 exists and path2 not. + Same as "check_file_exists(path1) and check_file_not_exists(path2)", just for convenience + + :param path1: the path to check if it exists + :param path2: the path to check if it doesn't exists + :returns: True if path1 exists and path2 doesn't, False otherwise + """ + return self.check_file_exists(path1) and self.check_file_not_exists(path2) + + def check_all_files_exists(self, *args): + """ + Check that all files passed as params exists. Intended to be used with few files. + + :param args: files to be checked + :returns: True if all files exist, False otherwise + """ + gen = (self.check_file_exists(i) for i in args) + return all(gen) + + def check_all_files_not_exists(self, *args): + """ + Check that all files passed as params don't exists. Intended to be used with few files. + + :param args: files to be checked + :returns: True if all files don't exist, False otherwise + """ + gen = (self.check_file_not_exists(i) for i in args) + return all(gen) + + def check_first_list_exists_second_list_not(self, pathlist1, pathlist2): + """ + Check that all files in the first list exist and all files in the second list don't + + :param pathlist1: list of files that must exist + :param pathlist2: list of files that mustn't exist + :returns: True if all files in the first list exist and all files in the second + list don't, False otherwise + """ + return self.check_all_files_exists(*pathlist1) and self.check_all_files_not_exists(*pathlist2) + +def pyocaction(username, password, async, method, *args, **kwargs): + """ + Run an action using the pyocclient. This method will create an ownCloud client and run + the method. A new client will be created each time. + + :param username: username for the client (for authentication purposes) + :param password: password for the client (for authentication purposes) + :param async: run the method async? True = async, False = sync + :param method: the name of the method to be run from the owncloud client (check pyocclient + Client object to know what methods you can use + :param *args: arguments that will be passed to the method + :param **kwargs: arguments that will be passed to the method. The extra keyword + 'pyocactiondebug' will enable debug in the Client object but won't be passed to the method + + :return: a tuple containing a Thread object and a Queue (to get the result of the called + method once it finish) if the async param is set to True, and the result of the method + if is set to False. + + examples: + `pyocaction(user, pass, False, 'put_file_contents', '/path/to/file', 'file content')` + `thread = pyocaction(user, pass, True, 'get_file', '/bigfile')` + + """ + oc_url = pyocclient_basic_url() + if kwargs.get('pyocactiondebug'): + client = owncloud.Client(oc_url, debug=True) + del kwargs['pyocactiondebug'] + else: + client = owncloud.Client(oc_url) + client.login(username, password) + + caller = getattr(client, method) + if async: + + def caller_wrapper(method, q, *args, **kwargs): + q.put(method(*args, **kwargs)) + + result_queue = Queue.Queue() + caller_wrapper_args = (caller, result_queue,) + args + thread = threading.Thread(target=caller_wrapper, args=caller_wrapper_args, kwargs=kwargs) + thread.start() + return (thread, result_queue) + else: + return caller(*args, **kwargs) + +def pyocclient_basic_url(): + """ + Return the url to be used by pyocclient-related functions / classes + """ + oc_protocol = 'https' if config.oc_ssl_enabled else 'http' + oc_url = oc_protocol + '://' + config.oc_server + '/' + config.oc_root + return oc_url