From 6b2e48ac3694842c058251e9953c44ea0e62a083 Mon Sep 17 00:00:00 2001 From: PintoGideon Date: Thu, 6 Jun 2024 10:44:53 -0400 Subject: [PATCH] fix: Update the code for running pipelines: --- src/components/NewLibrary/Cart.tsx | 87 ++-- src/components/NewLibrary/context/actions.ts | 2 + src/components/NewLibrary/context/index.tsx | 18 +- src/components/NewLibrary/useOperations.tsx | 453 +++++++++++-------- 4 files changed, 328 insertions(+), 232 deletions(-) diff --git a/src/components/NewLibrary/Cart.tsx b/src/components/NewLibrary/Cart.tsx index 6c10ad5d4..100646d40 100644 --- a/src/components/NewLibrary/Cart.tsx +++ b/src/components/NewLibrary/Cart.tsx @@ -51,11 +51,12 @@ export const Status = ({ item }: { item: SelectionPayload }) => { } if (type === "folder") { - const currentStatus = folderDownloadStatus[payload.data.id]; + const folderStatus = folderDownloadStatus?.[payload.data.id]; + const currentStatus = folderStatus?.status; return ( <> - {currentStatus === FolderDownloadTypes.finished ? ( + {currentStatus && currentStatus === FolderDownloadTypes.finished ? ( - - +
+ - - - - - + + + + + + + +
{ return { type: Types.SET_FOLDER_DOWNLOAD_STATUS, payload: { id: folder.data.id, status, + pipelineType, }, }; }; diff --git a/src/components/NewLibrary/context/index.tsx b/src/components/NewLibrary/context/index.tsx index d68da742b..2947e0dac 100644 --- a/src/components/NewLibrary/context/index.tsx +++ b/src/components/NewLibrary/context/index.tsx @@ -15,7 +15,10 @@ export interface LibraryState { [key: string]: DownloadTypes; }; folderDownloadStatus: { - [key: string]: FolderDownloadTypes; + [key: string]: { + status: FolderDownloadTypes; + pipelineType: string; + }; }; } @@ -82,7 +85,10 @@ export type LibraryPayload = { [Types.SET_FOLDER_DOWNLOAD_FROM_COOKIES]: { status: { - [key: string]: FolderDownloadTypes; + [key: string]: { + status: FolderDownloadTypes; + pipelineType: string; + }; }; }; @@ -102,6 +108,7 @@ export type LibraryPayload = { [Types.SET_FOLDER_DOWNLOAD_STATUS]: { id: number; status: FolderDownloadTypes; + pipelineType: string; }; [Types.CLEAR_DOWNLOAD_FILE_STATUS]: { @@ -164,13 +171,16 @@ export const libraryReducer = ( } case Types.SET_FOLDER_DOWNLOAD_STATUS: { - const { id, status } = action.payload; + const { id, status, pipelineType } = action.payload; return { ...state, folderDownloadStatus: { ...state.folderDownloadStatus, - [id]: status, + [id]: { + status, + pipelineType, + }, }, }; } diff --git a/src/components/NewLibrary/useOperations.tsx b/src/components/NewLibrary/useOperations.tsx index 874a52286..3699565f1 100644 --- a/src/components/NewLibrary/useOperations.tsx +++ b/src/components/NewLibrary/useOperations.tsx @@ -2,11 +2,12 @@ import type { Feed, FileBrowserFolder, FileBrowserFolderFile, + PipelineList, PluginInstance, } from "@fnndsc/chrisapi"; import { useMutation } from "@tanstack/react-query"; import axios from "axios"; -import { useContext, useState, useEffect } from "react"; +import { useContext, useState, useEffect, useCallback } from "react"; import ChrisAPIClient from "../../api/chrisapiclient"; import { MainRouterContext } from "../../App"; import { useTypedSelector } from "../../store/hooks"; @@ -33,7 +34,7 @@ const useOperations = () => { "PIPELINES", ]; - const [feedCreationError, setFeedCreatorError] = useState<{ + const [feedCreationError, setFeedCreationError] = useState<{ paths: string[]; error_message: string; }>({ @@ -49,16 +50,10 @@ const useOperations = () => { error_message: "", }); - const resetErrors = () => { - setFeedCreatorError({ - paths: [], - error_message: "", - }); - setDownloadError({ - paths: [], - error_message: "", - }); - }; + const resetErrors = useCallback(() => { + setFeedCreationError({ paths: [], error_message: "" }); + setDownloadError({ paths: [], error_message: "" }); + }, []); useEffect(() => { // recreate state whenever the component first mounts from session storage @@ -72,6 +67,7 @@ const useOperations = () => { const createFeed = () => { const invalidPaths: string[] = []; const validPaths: string[] = []; + state.selectedPaths.forEach(({ path }) => { if (cannotDownload.includes(path)) { invalidPaths.push(path); @@ -81,7 +77,7 @@ const useOperations = () => { }); if (invalidPaths.length > 0) { - setFeedCreatorError({ + setFeedCreationError({ ...feedCreationError, paths: [...feedCreationError.paths, ...invalidPaths], error_message: "Please avoid creating feeds with folders listed here:", @@ -92,62 +88,60 @@ const useOperations = () => { } }; - const clearFeed = () => { + const clearFeed = useCallback(() => { router.actions.clearFeedData(); - }; - - const clearPathSessionStorage = (path: string, id: number) => { - const copiedState = { ...state }; - const newSelectedPaths = copiedState.selectedPaths.filter((pathObj) => { - return pathObj.path !== path; - }); + }, [router.actions]); - // Delete ids - if (!isEmpty(FolderDownloadTypes)) { - delete copiedState.folderDownloadStatus[id]; - } + const clearPathSessionStorage = useCallback( + (path: string, id: number) => { + const copiedState = { ...state }; + const newSelectedPaths = copiedState.selectedPaths.filter( + (pathObj) => pathObj.path !== path, + ); - sessionStorage.setItem("selectedPaths", JSON.stringify(newSelectedPaths)); + if (!isEmpty(FolderDownloadTypes)) { + delete copiedState.folderDownloadStatus[id]; + } - sessionStorage.setItem( - "folderDownloadStatus", - JSON.stringify(copiedState.folderDownloadStatus), - ); - }; + localStorage.setItem("selectedPaths", JSON.stringify(newSelectedPaths)); + localStorage.setItem( + "folderDownloadStatus", + JSON.stringify(copiedState.folderDownloadStatus), + ); + }, + [state], + ); - const keepACopyInState = () => { + const keepACopyInState = useCallback(() => { if (!isEmpty(state.folderDownloadStatus)) { - sessionStorage.setItem( + localStorage.setItem( "folderDownloadStatus", JSON.stringify(state.folderDownloadStatus), ); } - if (!isEmpty(state.selectedPaths)) { - sessionStorage.setItem( + localStorage.setItem( "selectedPaths", JSON.stringify(state.selectedPaths), ); } - }; + }, [state.folderDownloadStatus, state.selectedPaths]); - const pickDownloadsFromSession = async ( - selectedPaths: any, - folderDownloadStatus: any, - ) => { - const downloadPromises = selectedPaths.map(async (userSelection: any) => { - const { type, payload } = userSelection; - if (type === "folder") { - // Check if current status exists - const currentStatus = folderDownloadStatus[payload.data.id]; - if (currentStatus) { + const pickDownloadsFromSession = useCallback( + async (selectedPaths: any, folderDownloadStatus: any) => { + const downloadPromises = selectedPaths.map(async (userSelection: any) => { + const { type, payload } = userSelection; + if (type === "folder") { + const { currentStatus, pipelineType } = + folderDownloadStatus[payload.data.id]; if (currentStatus === FolderDownloadTypes.zippingFolder) { - const data = await checkIfFeedExists(payload); + const data = await checkIfFeedExists(payload, pipelineType); if (!data || !data.feed || !data.createdInstance) { setDownloadErrorState( payload as FileBrowserFolder, payload.data.path, "Error downloading this folder/file", + pipelineType, ); } else { const { feed, createdInstance } = data; @@ -155,6 +149,7 @@ const useOperations = () => { downloadFolderStatus( payload as FileBrowserFolder, FolderDownloadTypes.zippingFolder, + pipelineType, ), ); await createPipelineDuringDownload( @@ -162,26 +157,29 @@ const useOperations = () => { payload.data.path, createdInstance, feed, + pipelineType, ); } } } - } - }); + }); - await Promise.all(downloadPromises); // Wait for all downloads to complete - }; + await Promise.all(downloadPromises); + }, + [dispatch], + ); - const recreateState = async () => { + const recreateState = useCallback(async () => { const client = ChrisAPIClient.getClient(); const folderDownloadStatus = JSON.parse( - sessionStorage.getItem("folderDownloadStatus") || "{}", + localStorage.getItem("folderDownloadStatus") || "{}", ); const selectedPaths = JSON.parse( - sessionStorage.getItem("selectedPaths") || "[]", + localStorage.getItem("selectedPaths") || "[]", ); - try { - if (isEmpty(state.selectedPaths) && !isEmpty(selectedPaths)) { + + if (isEmpty(state.selectedPaths) && !isEmpty(selectedPaths)) { + try { const updatedSelectedPaths = await Promise.all( selectedPaths.map(async (path: any) => { const url = new URL(path.payload.url); @@ -202,184 +200,260 @@ const useOperations = () => { await pickDownloadsFromSession( updatedSelectedPaths, folderDownloadStatus, - ); // Ensure this awaits the completion + ); } + } catch (error) { + console.error("Error", error); } - } catch (error) { - console.log("Error", error); } - }; - - const setDownloadErrorState = ( - payload: FileBrowserFolder, - path: string, - error_message: string, - ) => { - dispatch( - downloadFolderStatus( - payload as FileBrowserFolder, - FolderDownloadTypes.cancelled, - ), - ); - setDownloadError({ - ...downloadError, - paths: [...downloadError.paths, path], - error_message, - }); - }; - - const checkIfFeedExists = async (payload: FileBrowserFolder) => { - const path = payload.data.path; - const dircopy = await getPlugin("pl-dircopy"); - if (!dircopy) { - setDownloadErrorState( - payload as FileBrowserFolder, - path, - "Failed to find dircopy", + }, [ + dispatch, + pickDownloadsFromSession, + state.folderDownloadStatus, + state.selectedPaths, + ]); + + const setDownloadErrorState = useCallback( + ( + payload: FileBrowserFolder, + path: string, + error_message: string, + pipelineType: string, + ) => { + dispatch( + downloadFolderStatus( + payload as FileBrowserFolder, + FolderDownloadTypes.cancelled, + pipelineType, + ), ); - return { - feed: undefined, - folderName: "", - createdInstance: undefined, - }; - } - const client = ChrisAPIClient.getClient(); - // Check if the feed already exists with this name (Feed downloads cannot be repetitive) - const folderNameList = payload.data.path.split("/"); - const folderName = - folderNameList[folderNameList.length - 1] || payload.data.id; - - const feedExists = await client.getFeeds({ - name: `Library Download for ${folderName}`, - }); + setDownloadError({ + paths: [...downloadError.paths, path], + error_message, + }); + }, + [dispatch, downloadError.paths], + ); + + const checkIfFeedExists = useCallback( + async (payload: FileBrowserFolder, pipelineType: string) => { + const path = payload.data.path; + const dircopy = await getPlugin("pl-dircopy"); + if (!dircopy) { + setDownloadErrorState( + payload as FileBrowserFolder, + path, + "Failed to find dircopy", + pipelineType, + ); + return { feed: undefined, folderName: "", createdInstance: undefined }; + } + const client = ChrisAPIClient.getClient(); + const folderNameList = payload.data.path.split("/"); + const folderName = + folderNameList[folderNameList.length - 1] || payload.data.id; - let createdInstance: PluginInstance | undefined; - let feed: Feed | undefined; + const feedExists = await client.getFeeds({ + name: `Library ${pipelineType} for ${folderName}`, + }); - if (feedExists?.data) { - const feeds = feedExists.getItems(); - feed = feeds?.[0]; - const pluginInstances = await feed?.getPluginInstances({}); - const pluginInstancesItems = pluginInstances?.getItems(); - if (pluginInstancesItems) { - createdInstance = pluginInstancesItems[pluginInstancesItems.length - 1]; + let createdInstance: PluginInstance | undefined; + let feed: Feed | undefined; + + if (feedExists?.data) { + const feeds = feedExists.getItems(); + feed = feeds?.[0]; + const pluginInstances = await feed?.getPluginInstances({}); + const pluginInstancesItems = pluginInstances?.getItems(); + if (pluginInstancesItems) { + createdInstance = + pluginInstancesItems[pluginInstancesItems.length - 1]; + } + } else { + createdInstance = await client.createPluginInstance(dircopy.data.id, { + //@ts-ignore + dir: path, + }); + feed = (await createdInstance.getFeed()) as Feed; + await feed?.put({ name: `Library ${pipelineType} for ${folderName}` }); } - } else { - createdInstance = await client.createPluginInstance(dircopy.data.id, { - //@ts-ignore - dir: path, - }); - feed = (await createdInstance.getFeed()) as Feed; - } - return { feed, createdInstance, folderName }; - }; + return { feed, createdInstance, folderName }; + }, + [setDownloadErrorState], + ); const createFeedDuringDownload = async ( payload: FileBrowserFolder, path: string, + pipelineType: string, ) => { - const data = await checkIfFeedExists(payload as FileBrowserFolder); + const data = await checkIfFeedExists( + payload as FileBrowserFolder, + pipelineType, + ); if (!data) { setDownloadErrorState( payload as FileBrowserFolder, path, "Failed to create a Feed", + pipelineType, ); - } else { - await data?.feed?.put({ - name: `Library Download for ${data.folderName}`, - }); } return { feed: data?.feed, createdInstance: data?.createdInstance }; }; + const pollZipInstance = async ( + zipInstance: PluginInstance, + filePath: string, + path: string, + payload: FileBrowserFolder, + pipelineType: string, + ) => { + const client = ChrisAPIClient.getClient(); + const statusReq = await zipInstance.get(); + let status = statusReq.data.status; + + while (status !== "finishedSuccessfully") { + await new Promise((resolve) => setTimeout(resolve, 5000)); // Polling every 5 seconds + const statusReq = await zipInstance.get(); + status = statusReq.data.status; + } + + if (status === "finishedSuccessfully") { + const folderList = await client.getFileBrowserFolders({ + path: filePath, + }); + + if (!folderList) { + setDownloadErrorState( + payload as FileBrowserFolder, + path, + `Failed to find files under this ${filePath}`, + pipelineType, + ); + } + + const folders = folderList.getItems(); + + if (folders) { + const folder = folders[0]; + const files = await folder.getFiles(); + const fileItems = files.getItems(); + const fileToZip = fileItems[0]; + await downloadFile(fileToZip); + dispatch( + downloadFolderStatus( + payload as FileBrowserFolder, + FolderDownloadTypes.finished, + pipelineType, + ), + ); + } + } + }; + const createPipelineDuringDownload = async ( payload: FileBrowserFolder, path: string, createdInstance: PluginInstance, feed: Feed, + type, ) => { - const client = ChrisAPIClient.getClient(); - /** Zipping */ - const pipelineList = await client.getPipelines({ - name: "zip v20240311", - }); - - if (!pipelineList.data) { - setDownloadErrorState( - payload as FileBrowserFolder, - path, - "Please register the zip pipeline with the name v20240311", - ); - } + try { + const client = ChrisAPIClient.getClient(); - const pipelines = pipelineList.getItems(); + let pipelineList: PipelineList | null = null; + if (type === "download") { + /** Zipping */ + pipelineList = await client.getPipelines({ + name: "zip v20240311", + }); + } - if (pipelines && pipelines.length > 0) { - const pipeline = pipelines[0]; + if (type === "anonymize") { + pipelineList = await client.getPipelines({ + name: "DICOM anonymization simple v20230926", + }); + } - const { id } = pipeline.data; + if (!pipelineList || !pipelineList.data) { + setDownloadErrorState( + payload as FileBrowserFolder, + path, + "Please register the zip pipeline with the name v20240311", + type, + ); + } - //@ts-ignore - const workflow = await client.createWorkflow(id, { - previous_plugin_inst_id: createdInstance.data.id, - }); + if (pipelineList) { + const pipelines = pipelineList.getItems(); - const pluginInstancesList = await workflow.getPluginInstances({ - limit: 2, - }); - const pluginInstances = pluginInstancesList.getItems(); + // Check first if a zip pipeline was already added - if (pluginInstances && pluginInstances.length > 0) { - const zipInstance = pluginInstances[pluginInstances.length - 1]; - const statusReq = await zipInstance.get(); - let status = statusReq.data.status; + const pluginInstancesList = await feed.getPluginInstances({ + limit: 100, + }); - while (status !== "finishedSuccessfully") { - await new Promise((resolve) => setTimeout(resolve, 5000)); // Polling every 5 seconds - const statusReq = await zipInstance.get(); - status = statusReq.data.status; - } + const pluginInstances = pluginInstancesList.getItems(); - const filePath = `home/${username}/feeds/feed_${feed.data.id}/pl-dircopy_${createdInstance.data.id}/pl-pfdorun_${zipInstance.data.id}/data`; + if (pluginInstances && pluginInstances.length > 1) { + const zipInstance = pluginInstances[0]; - if (status === "finishedSuccessfully") { - const folderList = await client.getFileBrowserFolders({ - path: filePath, - }); + let filePath = ""; - if (!folderList) { - setDownloadErrorState( - payload as FileBrowserFolder, - path, - `Failed to find files under this ${filePath}`, - ); + if (type === "download") { + filePath = `home/${username}/feeds/feed_${feed.data.id}/pl-dircopy_${createdInstance.data.id}/pl-pfdorun_${zipInstance.data.id}/data`; } + if (type === "anonymize") { + const headerEditInstance = + pluginInstances[pluginInstances.length - 1]; + filePath = `home/${username}/feeds/feed_${feed.data.id}/pl-dircopy_${createdInstance.data.id}/pl-dicom_headeredit_${headerEditInstance.data.id}/pl-pfdorun_${zipInstance.data.id}/data`; + } + await pollZipInstance(zipInstance, filePath, path, payload, type); + } else { + // Plugin Instance does not exist and you need to create it. + if (pipelines && pipelines.length > 0) { + const pipeline = pipelines[0]; - const folders = folderList.getItems(); + const { id } = pipeline.data; - if (folders) { - const folder = folders[0]; - const files = await folder.getFiles(); - const fileItems = files.getItems(); - const fileToZip = fileItems[0]; - await downloadFile(fileToZip); - await feed.delete(); - dispatch( - downloadFolderStatus( - payload as FileBrowserFolder, - FolderDownloadTypes.finished, - ), - ); + //@ts-ignore + const workflow = await client.createWorkflow(id, { + previous_plugin_inst_id: createdInstance.data.id, + }); + + const pluginInstancesList = await workflow.getPluginInstances({ + limit: 3, + }); + const pluginInstances = pluginInstancesList.getItems(); + + if (pluginInstances && pluginInstances.length > 0) { + const zipInstance = pluginInstances[0]; + const headerEditInstance = pluginInstances[1]; + + let filePath = ""; + + if (type === "download") { + filePath = `home/${username}/feeds/feed_${feed.data.id}/pl-dircopy_${createdInstance.data.id}/pl-pfdorun_${zipInstance.data.id}/data`; + } + if (type === "anonymize") { + filePath = `home/${username}/feeds/feed_${feed.data.id}/pl-dircopy_${createdInstance.data.id}/pl-dicom_headeredit_${headerEditInstance.data.id}/pl-pfdorun_${zipInstance.data.id}/data`; + } + + await pollZipInstance(zipInstance, filePath, path, payload, type); + } } } } + } catch (error) { + // biome-ignore lint/complexity/noUselessCatch: + throw error; } }; - const handleDownload = async () => { + const handleDownload = async (pipelineType: string) => { const { selectedPaths } = state; const downloadPromises = selectedPaths.map(async (userSelection) => { @@ -409,6 +483,7 @@ const useOperations = () => { downloadFolderStatus( payload as FileBrowserFolder, FolderDownloadTypes.started, + pipelineType, ), ); @@ -420,12 +495,14 @@ const useOperations = () => { `Please don't zip folders in this list: ${cannotDownload.join( ", ", )}`, + pipelineType, ); } else { dispatch( downloadFolderStatus( payload as FileBrowserFolder, FolderDownloadTypes.creatingFeed, + pipelineType, ), ); @@ -433,6 +510,7 @@ const useOperations = () => { const data = await createFeedDuringDownload( payload as FileBrowserFolder, path, + pipelineType, ); if (!data || !data.feed || !data.createdInstance) { @@ -440,6 +518,7 @@ const useOperations = () => { payload as FileBrowserFolder, path, "Error downloading this folder/file", + pipelineType, ); } else { const { feed, createdInstance } = data; @@ -448,6 +527,7 @@ const useOperations = () => { downloadFolderStatus( payload as FileBrowserFolder, FolderDownloadTypes.zippingFolder, + pipelineType, ), ); @@ -456,6 +536,7 @@ const useOperations = () => { path, createdInstance, feed, + pipelineType, ); } } @@ -482,7 +563,7 @@ const useOperations = () => { }; const handleDownloadMutation = useMutation({ - mutationFn: () => handleDownload(), + mutationFn: ({ type }: { type: string }) => handleDownload(type), }); const handleDeleteMutation = useMutation({