Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement preserve.replicas when restaging files #234

Merged
merged 3 commits into from
Dec 19, 2023
Merged

implement preserve.replicas when restaging files #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f19ff7
[#232][#233] new tests for preserve replica set to true on other tier…
cookie33 Dec 14, 2023
612bc26
[#232] Work even if irods::storage_tiering::group has a higher unit t…
cookie33 Dec 12, 2023
7fd7b7e
[#233] implement preserve.replicas when restaging files
cookie33 Dec 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 87 additions & 2 deletions packaging/test_plugin_unified_storage_tiering.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,7 +746,7 @@ def tearDown(self):
admin_session.assert_icommand('iadmin rmresc ufs2')
admin_session.assert_icommand('iadmin rum')

def test_put_and_get(self):
def test_put(self):
with storage_tiering_configured_with_log():
IrodsController().restart(test_mode=True)
with session.make_session_for_existing_admin() as admin_session:
Expand All @@ -766,7 +766,7 @@ def test_put_and_get(self):
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', 'ufs1')

sleep(15)
# test prevent retier from preserved replica
# stage to tier 2, look for replica in tier 0 and tier 2
invoke_storage_tiering_rule()
admin_session.assert_icommand('iqstat', 'STDOUT_SINGLELINE', 'irods_policy_storage_tiering')

Expand All @@ -776,6 +776,91 @@ def test_put_and_get(self):
finally:
admin_session.assert_icommand('irm -f ' + filename)

def test_preserve_replicas_works_with_restage_when_replicas_exist_in_multiple_tiers__issue_232(self):
with storage_tiering_configured_with_log():
IrodsController().restart(test_mode=True)
with session.make_session_for_existing_admin() as admin_session:
filename = 'test_put_file'

try:
# make sure replicas stored on tier 2 are preserved
admin_session.assert_icommand('imeta add -R ufs2 irods::storage_tiering::preserve_replicas true')

# create test file and upload it
lib.create_local_testfile(filename)
admin_session.assert_icommand('iput -R ufs0 ' + filename)
admin_session.assert_icommand('ils -L ', 'STDOUT_SINGLELINE', 'rods')

# stage to tier 1, look for both replicas
sleep(6)
invoke_storage_tiering_rule()
admin_session.assert_icommand('iqstat', 'STDOUT_SINGLELINE', 'irods_policy_storage_tiering')

delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '0 ufs0')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '1 ufs1')

sleep(15)
# stage to tier 2, look for replica in tier 0 and tier 2
invoke_storage_tiering_rule()
admin_session.assert_icommand('iqstat', 'STDOUT_SINGLELINE', 'irods_policy_storage_tiering')

delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '0 ufs0')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '2 ufs2')

# test restage to tier 1 using replica from tier 0, look for replica in all tiers afterwards
admin_session.assert_icommand('iget -R ufs0 ' + filename + ' - ', 'STDOUT_SINGLELINE', 'TESTFILE')

sleep(15)
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '0 ufs0')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '3 ufs1')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '2 ufs2')


finally:
admin_session.assert_icommand('irm -f ' + filename)

def test_preserve_replicas_works_with_restage_when_replica_only_exists_in_last_tier__issue_233(self):
with storage_tiering_configured_with_log():
IrodsController().restart(test_mode=True)
with session.make_session_for_existing_admin() as admin_session:
filename = 'test_put_file'

try:
# make sure replicas stored on tier 1 are preserved and the minimum_restage_tier is not set
admin_session.assert_icommand('imeta add -R ufs1 irods::storage_tiering::preserve_replicas true')
admin_session.assert_icommand('imeta rm -R ufs1 irods::storage_tiering::minimum_restage_tier true')

# make sure tier 2 does not get involved
admin_session.assert_icommand('imeta rm -R ufs2 irods::storage_tiering::group example_group 2')

# create test file and upload it
lib.create_local_testfile(filename)
admin_session.assert_icommand('iput -R ufs0 ' + filename)
admin_session.assert_icommand('ils -L ', 'STDOUT_SINGLELINE', 'rods')

# stage to tier 1, look for both replicas
sleep(6)
invoke_storage_tiering_rule()
admin_session.assert_icommand('iqstat', 'STDOUT_SINGLELINE', 'irods_policy_storage_tiering')

delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '0 ufs0')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '1 ufs1')

# remove the object in tier 0
admin_session.assert_icommand('itrim -N1 -n0 ' + filename, 'STDOUT_SINGLELINE', 'Number of files trimmed = 1' )
alanking marked this conversation as resolved.
Show resolved Hide resolved

# test restage to tier 0, look for replica in tier 0 and tier 1
admin_session.assert_icommand('iget ' + filename + ' - ', 'STDOUT_SINGLELINE', 'TESTFILE')

sleep(15)
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '2 ufs0')
delay_assert_icommand(admin_session, 'ils -L ' + filename, 'STDOUT_SINGLELINE', '1 ufs1')


finally:
admin_session.assert_icommand('irm -f ' + filename)


class TestStorageTieringPluginObjectLimit(ResourceBase, unittest.TestCase):
def setUp(self):
super(TestStorageTieringPluginObjectLimit, self).setUp()
Expand Down
21 changes: 9 additions & 12 deletions storage_tiering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,33 +753,31 @@ namespace irods {

} // get_replica_number_for_resource

std::string storage_tiering::get_group_name_by_replica_number(
std::string storage_tiering::get_group_name_for_object(
rcComm_t* _comm,
const std::string& _attribute_name,
const std::string& _object_path,
const std::string& _replica_number) {
const std::string& _object_path) {
boost::filesystem::path p{_object_path};
std::string data_name = p.filename().string();
std::string coll_name = p.parent_path().string();

std::string qstr{boost::str(
boost::format("SELECT META_DATA_ATTR_VALUE WHERE DATA_NAME = '%s' AND COLL_NAME = '%s' AND META_DATA_ATTR_NAME = '%s' AND META_DATA_ATTR_UNITS = '%s'")
boost::format("SELECT META_DATA_ATTR_VALUE WHERE DATA_NAME = '%s' AND COLL_NAME = '%s' AND META_DATA_ATTR_NAME = '%s'")
% data_name
% coll_name
% _attribute_name
% _replica_number)};
% _attribute_name)};

query<rcComm_t> qobj{_comm, qstr};

if(qobj.size() == 0) {
THROW(
CAT_NO_ROWS_FOUND,
"failed to fetch group name by resource and replica number");
"failed to fetch group name by object and resource");
}

return qobj.front()[0];

} // get_group_name_by_replica_number
} // get_group_name_for_object

void storage_tiering::migrate_object_to_minimum_restage_tier(
const std::string& _object_path,
Expand All @@ -792,11 +790,10 @@ namespace irods {
comm_,
_object_path,
_source_resource);
const auto group_name = get_group_name_by_replica_number(
const auto group_name = get_group_name_for_object(
comm_,
config_.group_attribute,
_object_path,
source_replica_number);
_object_path);
const auto low_tier_resource_name = get_restage_tier_resource_name(
comm_,
group_name);
Expand All @@ -817,7 +814,7 @@ namespace irods {
_source_resource,
low_tier_resource_name,
get_verification_for_resc(comm_, low_tier_resource_name),
false,
get_preserve_replicas_for_resc(comm_, _source_resource),
get_data_movement_parameters_for_resource(comm_, _source_resource));
}
catch(const exception& _e) {
Expand Down
5 changes: 2 additions & 3 deletions storage_tiering.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,10 @@ namespace irods {
const std::string& _object_path,
const std::string& _resource_name);

std::string get_group_name_by_replica_number(
std::string get_group_name_for_object(
rcComm_t* _comm,
const std::string& _attribute_name,
const std::string& _object_path,
const std::string& _replica_number);
const std::string& _object_path);

resource_index_map
get_resource_map_for_group(
Expand Down
Loading