diff --git a/.scripts/download_zenodo.py b/.scripts/download_zenodo.py index 7201d7fd..88de13f8 100755 --- a/.scripts/download_zenodo.py +++ b/.scripts/download_zenodo.py @@ -4,7 +4,6 @@ import urllib.request import hashlib import sys -import os from pathlib import Path @@ -19,15 +18,15 @@ def calculate_md5(filename): def download_zenodo_files(output_dir: Path): """ - Download all files from Zenodo record 14652312 and verify their checksums. + Download all files from Zenodo record 14833590 and verify their checksums. Args: output_dir: Directory where files should be downloaded """ try: - print("Fetching files from Zenodo record 14652312...") + print("Fetching files from Zenodo record 14833590...") with urllib.request.urlopen( - "https://zenodo.org/api/records/14652312" + "https://zenodo.org/api/records/14833590" ) as response: data = json.loads(response.read()) diff --git a/pyproject.toml b/pyproject.toml index 5263c9f4..27e53c35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,8 +72,6 @@ addopts = [ "-vv", "-ra", "-q", - "--benchmark-sort=mean", - "--benchmark-columns=mean", "--tb=native", "--cov-report=term", "--cov-report=xml:cov.xml", diff --git a/zenodo-tests/conftest.py b/zenodo-tests/conftest.py index 7f8c40c0..1fd0cd5a 100644 --- a/zenodo-tests/conftest.py +++ b/zenodo-tests/conftest.py @@ -2,6 +2,8 @@ import cupy as cp import numpy as np import pytest +import gc + def pytest_addoption(parser): parser.addoption( @@ -31,8 +33,8 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_perf) -#CUR_DIR = os.path.abspath(os.path.dirname(__file__)) -CUR_DIR = "/dls/science/users/kjy41806/zenodo-tests/" +CUR_DIR = os.path.abspath(os.path.dirname(__file__)) + @pytest.fixture(scope="session") def test_data_path(): @@ -119,6 +121,22 @@ def i13_dataset2(i13_dataset2_file): ) +@pytest.fixture(scope="session") +def i13_dataset3_file(test_data_path): + in_file = os.path.join(test_data_path, "i13_dataset3.npz") + return np.load(in_file) + + +@pytest.fixture +def i13_dataset3(i13_dataset3_file): + return ( + cp.asarray(i13_dataset3_file["projdata"]), + i13_dataset3_file["angles"], + cp.asarray(i13_dataset3_file["flats"]), + cp.asarray(i13_dataset3_file["darks"]), + ) + + @pytest.fixture(scope="session") def k11_dataset1_file(test_data_path): in_file = os.path.join(test_data_path, "k11_dataset1.npz") @@ -153,8 +171,15 @@ def geant4_dataset1(geant4_dataset1_file): @pytest.fixture def ensure_clean_memory(): + gc.collect() cp.get_default_memory_pool().free_all_blocks() cp.get_default_pinned_memory_pool().free_all_blocks() yield None cp.get_default_memory_pool().free_all_blocks() cp.get_default_pinned_memory_pool().free_all_blocks() + + +def force_clean_gpu_memory(): + gc.collect() + cp.get_default_memory_pool().free_all_blocks() + cp.get_default_pinned_memory_pool().free_all_blocks() diff --git a/zenodo-tests/test_misc/test_morph.py b/zenodo-tests/test_misc/test_morph.py index 0424edac..15df304a 100644 --- a/zenodo-tests/test_misc/test_morph.py +++ b/zenodo-tests/test_misc/test_morph.py @@ -6,20 +6,22 @@ from numpy.testing import assert_allclose from httomolibgpu.misc.morph import sino_360_to_180 from httomolibgpu.prep.normalize import normalize +from conftest import force_clean_gpu_memory -def test_sino_360_to_180_i13_dataset1(i13_dataset1, ensure_clean_memory): - +def test_sino_360_to_180_i13_dataset1(i13_dataset1): projdata = i13_dataset1[0] flats = i13_dataset1[2] darks = i13_dataset1[3] del i13_dataset1 - + data_normalised = normalize(projdata, flats, darks, minus_log=True) del flats, darks, projdata - ensure_clean_memory + force_clean_gpu_memory() - stiched_data_180degrees = sino_360_to_180(data_normalised, overlap = 473.822265625, rotation = 'right') + stiched_data_180degrees = sino_360_to_180( + data_normalised, overlap=473.822265625, rotation="right" + ) stiched_data_180degrees = stiched_data_180degrees.get() assert_allclose(np.sum(stiched_data_180degrees), 28512826.0, rtol=1e-07, atol=1e-6) @@ -28,4 +30,22 @@ def test_sino_360_to_180_i13_dataset1(i13_dataset1, ensure_clean_memory): assert stiched_data_180degrees.flags.c_contiguous +def test_sino_360_to_180_i13_dataset3(i13_dataset3): + projdata = i13_dataset3[0] + flats = i13_dataset3[2] + darks = i13_dataset3[3] + del i13_dataset3 + + data_normalised = normalize(projdata, flats, darks, minus_log=True) + del flats, darks, projdata + force_clean_gpu_memory() + stiched_data_180degrees = sino_360_to_180( + data_normalised, overlap=438.173828, rotation="left" + ) + stiched_data_180degrees = stiched_data_180degrees.get() + + assert_allclose(np.sum(stiched_data_180degrees), 24865750.0, rtol=1e-07, atol=1e-6) + assert stiched_data_180degrees.shape == (3000, 3, 4682) + assert stiched_data_180degrees.dtype == np.float32 + assert stiched_data_180degrees.flags.c_contiguous diff --git a/zenodo-tests/test_recon/test_algorithm.py b/zenodo-tests/test_recon/test_algorithm.py index 8982300a..c8de7542 100644 --- a/zenodo-tests/test_recon/test_algorithm.py +++ b/zenodo-tests/test_recon/test_algorithm.py @@ -3,19 +3,23 @@ import pytest from cupy.cuda import nvtx import time +from math import isclose from httomolibgpu.prep.normalize import normalize from httomolibgpu.recon.algorithm import ( FBP, LPRec, ) +from httomolibgpu.misc.morph import sino_360_to_180 from numpy.testing import assert_allclose import time import pytest from cupy.cuda import nvtx +from conftest import force_clean_gpu_memory -def test_reconstruct_FBP_i12_dataset1(i12_dataset1, ensure_clean_memory): +def test_reconstruct_FBP_i12_dataset1(i12_dataset1): + force_clean_gpu_memory() projdata = i12_dataset1[0] angles = i12_dataset1[1] flats = i12_dataset1[2] @@ -24,13 +28,13 @@ def test_reconstruct_FBP_i12_dataset1(i12_dataset1, ensure_clean_memory): data_normalised = normalize(projdata, flats, darks, minus_log=True) del flats, darks, projdata - ensure_clean_memory + force_clean_gpu_memory() recon_data = FBP( data_normalised, np.deg2rad(angles), center=1253.75, - filter_freq_cutoff=0.35, + filter_freq_cutoff=0.35, ) assert recon_data.flags.c_contiguous recon_data = recon_data.get() @@ -38,8 +42,50 @@ def test_reconstruct_FBP_i12_dataset1(i12_dataset1, ensure_clean_memory): assert recon_data.dtype == np.float32 assert recon_data.shape == (2560, 50, 2560) + +def test_reconstruct_LP_REC_i13_dataset1(i13_dataset1): + force_clean_gpu_memory() + projdata = i13_dataset1[0] + angles = i13_dataset1[1] + flats = i13_dataset1[2] + darks = i13_dataset1[3] + del i13_dataset1 + + data_normalised = normalize(projdata, flats, darks, minus_log=True) + del flats, darks, projdata + force_clean_gpu_memory() + + stiched_data_180degrees = sino_360_to_180( + data_normalised[:, 2:3, :], overlap=473.822265625, rotation="right" + ) + del data_normalised + force_clean_gpu_memory() + + # GPU archetictures older than 5.3 wont accept the data larger than + # (4096, 4096, 4096), while the newer ones can accept (16384 x 16384 x 16384) + + # recon_data = FBP( + # stiched_data_180degrees, + # np.deg2rad(angles[0:3000]), + # center=2322, + # filter_freq_cutoff=0.35, + # ) + recon_data = LPRec( + data=stiched_data_180degrees, + angles=np.deg2rad(angles[0:3000]), + center=2322.08, + ) + + assert recon_data.flags.c_contiguous + recon_data = recon_data.get() + assert isclose(np.sum(recon_data), 620.8565, abs_tol=10**-4) + assert recon_data.dtype == np.float32 + assert recon_data.shape == (4646, 1, 4646) + + @pytest.mark.perf -def test_FBP_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): +def test_FBP_performance_i13_dataset2(i13_dataset2): + force_clean_gpu_memory() dev = cp.cuda.Device() projdata = i13_dataset2[0] angles = i13_dataset2[1] @@ -49,7 +95,7 @@ def test_FBP_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): data_normalised = normalize(projdata, flats, darks, minus_log=True) del flats, darks, projdata - ensure_clean_memory + force_clean_gpu_memory() # cold run first FBP( @@ -57,17 +103,17 @@ def test_FBP_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): np.deg2rad(angles), center=1253.75, filter_freq_cutoff=0.35, - ) + ) dev.synchronize() start = time.perf_counter_ns() nvtx.RangePush("Core") for _ in range(10): FBP( - data_normalised, - np.deg2rad(angles), - center=1286.25, - filter_freq_cutoff=0.35, + data_normalised, + np.deg2rad(angles), + center=1286.25, + filter_freq_cutoff=0.35, ) nvtx.RangePop() dev.synchronize() @@ -76,8 +122,8 @@ def test_FBP_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): assert "performance in ms" == duration_ms -def test_reconstruct_LPREC_i13_dataset2(i13_dataset2, ensure_clean_memory): - +def test_reconstruct_LPREC_i13_dataset2(i13_dataset2): + force_clean_gpu_memory() projdata = i13_dataset2[0] angles = i13_dataset2[1] flats = i13_dataset2[2] @@ -86,8 +132,7 @@ def test_reconstruct_LPREC_i13_dataset2(i13_dataset2, ensure_clean_memory): data_normalised = normalize(projdata, flats, darks, minus_log=True) del flats, darks, projdata - ensure_clean_memory - + force_clean_gpu_memory() recon_data = LPRec( data=data_normalised, @@ -97,13 +142,13 @@ def test_reconstruct_LPREC_i13_dataset2(i13_dataset2, ensure_clean_memory): assert recon_data.flags.c_contiguous recon_data = recon_data.get() - assert_allclose(np.sum(recon_data), 2044.9531, rtol=1e-07, atol=1e-6) + assert isclose(np.sum(recon_data), 2044.953, abs_tol=10**-3) assert recon_data.dtype == np.float32 assert recon_data.shape == (2560, 10, 2560) @pytest.mark.perf -def test_LPREC_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): +def test_LPREC_performance_i13_dataset2(i13_dataset2): dev = cp.cuda.Device() projdata = i13_dataset2[0] angles = i13_dataset2[1] @@ -113,7 +158,7 @@ def test_LPREC_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): data_normalised = normalize(projdata, flats, darks, minus_log=True) del flats, darks, projdata - ensure_clean_memory + force_clean_gpu_memory() # cold run first LPRec( @@ -127,12 +172,46 @@ def test_LPREC_performance_i13_dataset2(i13_dataset2, ensure_clean_memory): nvtx.RangePush("Core") for _ in range(10): LPRec( - data=data_normalised, - angles=np.deg2rad(angles), - center=1286.25, + data=data_normalised, + angles=np.deg2rad(angles), + center=1286.25, ) nvtx.RangePop() dev.synchronize() duration_ms = float(time.perf_counter_ns() - start) * 1e-6 / 10 - assert "performance in ms" == duration_ms \ No newline at end of file + assert "performance in ms" == duration_ms + + +def test_reconstruct_FBP_i13_dataset3(i13_dataset3): + force_clean_gpu_memory() + projdata = i13_dataset3[0] + angles = i13_dataset3[1] + flats = i13_dataset3[2] + darks = i13_dataset3[3] + del i13_dataset3 + + data_normalised = normalize(projdata, flats, darks, minus_log=True) + del flats, darks, projdata + force_clean_gpu_memory() + + stiched_data_180degrees = sino_360_to_180( + data_normalised, overlap=438.173828, rotation="left" + ) + force_clean_gpu_memory() + + # GPU archetictures older than 5.3 wont accept the data larger than + # (4096, 4096, 4096), while the newer ones can accept (16384 x 16384 x 16384) + + recon_data = FBP( + stiched_data_180degrees, + np.deg2rad(angles[0:3000]), + center=2341, + filter_freq_cutoff=0.35, + ) + + assert recon_data.flags.c_contiguous + recon_data = recon_data.get() + # assert_allclose(np.sum(recon_data), 620.85657, rtol=1e-07, atol=1e-6) + assert recon_data.dtype == np.float32 + assert recon_data.shape == (4682, 3, 4682) diff --git a/zenodo-tests/test_recon/test_rotation.py b/zenodo-tests/test_recon/test_rotation.py index 6c7612c8..89ea1176 100644 --- a/zenodo-tests/test_recon/test_rotation.py +++ b/zenodo-tests/test_recon/test_rotation.py @@ -21,7 +21,9 @@ def test_center_vo_i12_dataset1(i12_dataset1, ensure_clean_memory): ensure_clean_memory mid_slice = data_normalised.shape[1] // 2 - cor = find_center_vo(data_normalised[:, mid_slice, :]) + cor = find_center_vo( + data_normalised[:, mid_slice, :], smin=-100, smax=100, step=0.25 + ) assert cor == 1253.75 assert cor.dtype == np.float32 @@ -47,7 +49,7 @@ def test_center_vo_i12_dataset1_performance(i12_dataset1, ensure_clean_memory): start = time.perf_counter_ns() nvtx.RangePush("Core") for _ in range(10): - find_center_vo(data_normalised[:, mid_slice, :]) + find_center_vo(data_normalised[:, mid_slice, :], smin=-100, smax=100, step=0.25) nvtx.RangePop() dev.synchronize() duration_ms = float(time.perf_counter_ns() - start) * 1e-6 / 10 @@ -68,7 +70,9 @@ def test_center_vo_i12_dataset2(i12_dataset2, ensure_clean_memory): ensure_clean_memory mid_slice = data_normalised.shape[1] // 2 - cor = find_center_vo(data_normalised[:, mid_slice, :]) + cor = find_center_vo( + data_normalised[:, mid_slice, :], smin=-100, smax=100, step=0.25 + ) assert cor == 1197.75 assert cor.dtype == np.float32 @@ -84,7 +88,9 @@ def test_center_vo_average_i12_dataset2(i12_dataset2, ensure_clean_memory): del flats, darks, projdata ensure_clean_memory - cor = find_center_vo(data_normalised[:, 10:25, :], average_radius=5) + cor = find_center_vo( + data_normalised[:, 10:25, :], average_radius=5, smin=-100, smax=100, step=0.25 + ) assert cor == 1199.25 assert cor.dtype == np.float32 @@ -133,7 +139,7 @@ def test_center_360_i13_dataset1(i13_dataset1, ensure_clean_memory): cor, overlap, side, overlap_position = find_center_360(data_normalised) assert int(cor) == 2322 - assert int(overlap) == 473 # actual 473.822265625 + assert int(overlap) == 473 # actual 473.822265625 assert cor.dtype == np.float64 @@ -151,12 +157,42 @@ def test_center_vo_i13_dataset2(i13_dataset2, ensure_clean_memory): ensure_clean_memory mid_slice = data_normalised.shape[1] // 2 - cor = find_center_vo(data_normalised[:, mid_slice, :]) + cor = find_center_vo( + data_normalised[:, mid_slice, :], smin=-100, smax=100, step=0.25 + ) assert cor == 1286.25 assert cor.dtype == np.float32 +# ----------------------------------------------------------# +# i13_dataset3 tests +# 360 degrees dataset +def test_center_360_i13_dataset3(i13_dataset3, ensure_clean_memory): + projdata = i13_dataset3[0] + flats = i13_dataset3[2] + darks = i13_dataset3[3] + del i13_dataset3 + + data_normalised = normalize(projdata, flats, darks, minus_log=False) + del flats, darks, projdata + + ensure_clean_memory + cor, overlap, side, overlap_position = find_center_360( + data_normalised, + ind=1, + win_width=50, + side=0, + denoise=True, + norm=True, + use_overlap=True, + ) + + assert int(cor) == 218 # actual 218.08 (not correct CoR actually, should be 2341) + assert int(overlap) == 438 # actual 438.173828 + assert cor.dtype == np.float64 + + # ----------------------------------------------------------# # geant4_dataset1 tests def test_center_vo_geant4_dataset1(geant4_dataset1, ensure_clean_memory): @@ -189,7 +225,9 @@ def test_center_vo_k11_dataset1(k11_dataset1, ensure_clean_memory): ensure_clean_memory mid_slice = data_normalised.shape[1] // 2 - cor = find_center_vo(data_normalised[:, mid_slice, :]) + cor = find_center_vo( + data_normalised[:, mid_slice, :], smin=-100, smax=100, step=0.25 + ) assert cor == 1269.25 assert cor.dtype == np.float32