Feature: support for shared downloads directory (#295)

* Feature: support for shared downloads directory

This feature is especially relevant for users with several clients running simultaneously within the same computer (computers running several GPUs or any combination of GPUs + CPUs) or multiple computers running in a network. The feature allows the user to specify a shared downloads directory via the new -shared-downloads-dir client option. All the clients using that option will share the same binaries and scene files.

How it works?
The first client downloading a binary or scene will save the file in the directory especified in -shared-downloads-directory. The rest of the clients using the same option will wait until the file has been downloaded. Once the file has been downloaded, it will be ready for the rest of the clients.

This feature is especially relevant for users with metered/slow connections or multiple computers/clients that don't want to download the same binary/file multiple times.

IMPORTANT: All the clients intended to share the binaries and scenes must execute the client with the same -shared-downloads-dir parameter.

src/com/sheepit/client/Client.java

@@ -24,16 +24,21 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.Observable;
 import java.util.Observer;
+import java.util.Random;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 
 import com.sheepit.client.Error.ServerCode;
 import com.sheepit.client.Error.Type;
@@ -783,22 +788,77 @@ public Error.Type work(final Job ajob) {
 	}
 
 	protected Error.Type downloadSceneFile(Job ajob_) throws FermeExceptionNoSpaceLeftOnDevice {
-		return this.downloadFile(ajob_, ajob_.getSceneArchivePath(), ajob_.getSceneMD5(),
+		return this.downloadFile(ajob_, ajob_.getRequiredSceneArchivePath(), ajob_.getSceneMD5(),
 				String.format("%s?type=job&job=%s", this.server.getPage("download-archive"), ajob_.getId()), "project");
 	}
 
 	protected Error.Type downloadExecutable(Job ajob) throws FermeExceptionNoSpaceLeftOnDevice {
-		return this.downloadFile(ajob, ajob.getRendererArchivePath(), ajob.getRendererMD5(),
+		return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
 				String.format("%s?type=binary&job=%s", this.server.getPage("download-archive"), ajob.getId()), "renderer");
 	}
 
 	private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws FermeExceptionNoSpaceLeftOnDevice {
 		File local_path_file = new File(local_path);
 		String update_ui = "Downloading " + download_type;
 
-		if (local_path_file.exists() == true) {
-			this.gui.status("Reusing cached " + download_type);
-			return Type.OK;
+		int remaining = 1800000; // 30 minutes max timeout
+
+		try {
+			// If the client is using a shared cache then introduce some random delay to minimise race conditions on the partial file creation on multiple
+			// instances of a client (when started with a script or rendering a recently downloaded scene)
+			if (configuration.getSharedDownloadsDirectory() != null) {
+				Thread.sleep((new Random().nextInt(9) + 1) * 1000);
+			}
+
+			// For a maximum of 30 minutes
+			do {
+				// if the binary or scene already exists in the cache
+				if (local_path_file.exists() == true) {
+					this.gui.status("Reusing cached " + download_type);
+					return Type.OK;
+				}
+				// if the binary or scene is being downloaded by another client
+				else if (new File(local_path + ".partial").exists()) {
+					// Wait and check every second for file download completion but only update the GUI every 10 seconds to minimise CPU load
+					if (remaining % 10000 == 0) {
+						this.gui.status(String.format("Another client is downloading the %s. Cancel in %dmin %ds",
+							download_type,
+							TimeUnit.MILLISECONDS.toMinutes(remaining),
+							TimeUnit.MILLISECONDS.toSeconds(remaining) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(remaining))
+						));
+					}
+				}
+				else {
+					// The file doesn't yet existing not is being downloaded by another client, so immediately create the file with zero bytes to allow early
+					// detection by other concurrent clients and start downloading process
+					try {
+						File file = new File(local_path + ".partial");
+						file.createNewFile();
+						file.deleteOnExit();    // if the client crashes, the temporary file will be removed
+					} catch (IOException e) {
+						StringWriter sw = new StringWriter();
+						e.printStackTrace(new PrintWriter(sw));
+						this.log.error("Client::DownloadFile Unable to create .partial temp file for binary/scene " + local_path);
+						this.log.error("Client::DownloadFile Exception " + e + " stacktrace " + sw.toString());
+					}
+
+					break;
+				}
+
+				// Reduce 1 second the waiting time
+				Thread.sleep(1000);
+				remaining -= 1000;
+			} while (remaining > 0);
+		}
+		catch (InterruptedException e) {
+			log.debug("Error in the thread wait. Exception " + e.getMessage());
+		}
+		finally {
+			// If we have reached the timeout (30 minutes trying to download the client) delete the partial downloaded copy and try to download again
+			if (remaining <= 0) {
+				log.debug("ERROR while waiting for download to finish in another client. Deleting the partial file and downloading a fresh copy now!.");
+				new File(local_path + ".partial").delete();
+			}
 		}
 
 		this.gui.status(String.format("Downloading %s", download_type), 0, 0);
@@ -869,18 +929,28 @@ protected void removeSceneDirectory(Job ajob) {
 
 	protected int prepareWorkingDirectory(Job ajob) throws FermeExceptionNoSpaceLeftOnDevice {
 		int ret;
+		String bestRendererArchive = ajob.getRequiredRendererArchivePath();
 		String renderer_archive = ajob.getRendererArchivePath();
 		String renderer_path = ajob.getRendererDirectory();
 		File renderer_path_file = new File(renderer_path);
 
-		if (renderer_path_file.exists()) {
-			// Directory already exists -> do nothing
+		if (!new File(renderer_archive).exists()) {
+			this.gui.status("Copying renderer from shared downloads directory");
+
+			try {
+				Files.copy(Paths.get(bestRendererArchive), Paths.get(renderer_archive), StandardCopyOption.REPLACE_EXISTING);
+			}
+			catch (IOException e) {
+				this.gui.error("Error while copying renderer from shared downloads directory to working dir");
+			}
 		}
-		else {
-			this.gui.status("Extracting renderer");
+
+		if (!renderer_path_file.exists()) {
 			// we create the directory
 			renderer_path_file.mkdir();
 
+			this.gui.status("Extracting renderer");
+
 			// unzip the archive
 			ret = Utils.unzipFileIntoDirectory(renderer_archive, renderer_path, null, log);
 			if (ret != 0) {
@@ -899,18 +969,28 @@ protected int prepareWorkingDirectory(Job ajob) throws FermeExceptionNoSpaceLeft
 			}
 		}
 
+		String bestSceneArchive = ajob.getRequiredSceneArchivePath();
 		String scene_archive = ajob.getSceneArchivePath();
 		String scene_path = ajob.getSceneDirectory();
 		File scene_path_file = new File(scene_path);
 
-		if (scene_path_file.exists()) {
-			// Directory already exists -> do nothing
+		if (!new File(scene_archive).exists()) {
+			this.gui.status("Copying scene from common directory");
+
+			try {
+				Files.copy(Paths.get(bestSceneArchive), Paths.get(scene_archive), StandardCopyOption.REPLACE_EXISTING);
+			}
+			catch (IOException e) {
+				this.gui.error("Error while copying scene from common directory to working dir");
+			}
 		}
-		else {
-			this.gui.status("Extracting project");
+
+		if (!scene_path_file.exists()) {
 			// we create the directory
 			scene_path_file.mkdir();
 
+			this.gui.status("Extracting project");
+
 			// unzip the archive
 			ret = Utils.unzipFileIntoDirectory(scene_archive, scene_path, ajob.getPassword(), log);
 			if (ret != 0) {

src/com/sheepit/client/Configuration.java

@@ -43,6 +43,7 @@ public enum ComputeType {
 
 	private String configFilePath;
 	private File workingDirectory;
+	private File sharedDownloadsDirectory;
 	private File storageDirectory; // for permanent storage (binary archive)
 	private boolean userHasSpecifiedACacheDir;
 	private String static_exeDirName;
@@ -87,6 +88,7 @@ public Configuration(File cache_dir_, String login_, String password_) {
 		this.userHasSpecifiedACacheDir = false;
 		this.detectGPUs = true;
 		this.workingDirectory = null;
+		this.sharedDownloadsDirectory = null;
 		this.storageDirectory = null;
 		this.setCacheDir(cache_dir_);
 		this.printLog = false;
@@ -147,6 +149,13 @@ public void setCacheDir(File cache_dir_) {
 			this.storageDirectory.mkdirs();
 		}
 
+		if (this.sharedDownloadsDirectory != null) {
+			this.sharedDownloadsDirectory.mkdirs();
+
+			if (!this.sharedDownloadsDirectory.exists()) {
+				System.err.println("Configuration::setCacheDir Unable to create common directory " + this.sharedDownloadsDirectory.getAbsolutePath());
+			}
+		}
 	}
 
 	public void setStorageDir(File dir) {
@@ -253,6 +262,12 @@ public List<File> getLocalCacheFiles() {
 				files.addAll(Arrays.asList(filesInDirectory));
 			}
 		}
+		if (this.sharedDownloadsDirectory != null) {
+			File[] filesInDirectory = this.sharedDownloadsDirectory.listFiles();
+			if (filesInDirectory != null) {
+				files.addAll(Arrays.asList(filesInDirectory));
+			}
+		}
 
 		for (File file : files) {
 			if (file.isFile()) {

src/com/sheepit/client/Job.java

@@ -141,6 +141,15 @@ public String getRendererDirectory() {
 		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererMD5;
 	}
 
+	public String getRequiredRendererArchivePath() {
+		if (configuration.getSharedDownloadsDirectory() != null) {
+			return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
+		}
+		else {
+			return getRendererArchivePath();
+		}
+	}
+
 	public String getRendererPath() {
 		return getRendererDirectory() + File.separator + OS.getOS().getRenderBinaryPath();
 	}
@@ -149,6 +158,15 @@ public String getRendererArchivePath() {
 		return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
 	}
 
+	public String getRequiredSceneArchivePath() {
+		if (configuration.getSharedDownloadsDirectory() != null) {
+			return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip";
+		}
+		else {
+			return getSceneArchivePath();
+		}
+	}
+
 	public String getSceneDirectory() {
 		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5;
 	}

src/com/sheepit/client/Server.java

@@ -433,7 +433,7 @@ public Error.Type HTTPGetFile(String url_, String destination_, Gui gui_, String
 			}
 
 			is = response.body().byteStream();
-			output = new FileOutputStream(destination_);
+			output = new FileOutputStream(destination_ + ".partial");
 
 			long size = response.body().contentLength();
 			byte[] buffer = new byte[8 * 1024];
@@ -488,6 +488,17 @@ else if (this.client.getRenderingJob().isUserBlockJob()) {
 					output.close();
 				}
 
+				File downloadedFile = new File(destination_ + ".partial");
+
+				if (downloadedFile.exists()) {
+					// Rename file (or directory)
+					boolean success = downloadedFile.renameTo(new File(destination_));
+
+					if (!success) {
+						this.log.debug(String.format("Server::HTTPGetFile Error trying to rename the downloaded file to final name (%s)", destination_));
+					}
+				}
+
 				if (is != null) {
 					is.close();
 				}
@@ -629,6 +640,14 @@ private void handleFileMD5DeleteDocument(List<FileMD5> fileMD5s) {
 					File file_to_delete = new File(path + ".zip");
 					file_to_delete.delete();
 					Utils.delete(new File(path));
+
+					// If we are using a shared downloads directory, then delete the file from the shared downloads directory as well :)
+					if (this.user_config.getSharedDownloadsDirectory() != null) {
+						String commonCacheFile = this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5();
+						this.log.debug("Server::handleFileMD5DeleteDocument delete common file " + commonCacheFile + ".zip");
+						file_to_delete = new File(commonCacheFile + ".zip");
+						file_to_delete.delete();
+					}
 				}
 			}
 		}

src/com/sheepit/client/standalone/Worker.java

@@ -61,6 +61,8 @@ public class Worker {
 
 	@Option(name = "-cache-dir", usage = "Cache/Working directory. Caution, everything in it not related to the render-farm will be removed", metaVar = "/tmp/cache", required = false) private String cache_dir = null;
 
+	@Option(name = "-shared-zip", usage = "Shared directory for downloaded binaries and scenes. Useful when running two or more clients in the same computer/network to download once and render many times. IMPORTANT: This option and value must be identical in ALL clients sharing the directory.", required = false) private String sharedDownloadsDir = null;
+
 	@Option(name = "-gpu", usage = "Name of the GPU used for the render, for example CUDA_0 for Nvidia or OPENCL_0 for AMD/Intel card", metaVar = "CUDA_0", required = false) private String gpu_device = null;
 
 	@Option(name = "--no-gpu", usage = "Don't detect GPUs", required = false) private boolean no_gpu_detection = false;
@@ -130,6 +132,15 @@ public void doMain(String[] args) {
 		config.setUsePriority(priority);
 		config.setDetectGPUs(!no_gpu_detection);
 
+		if (sharedDownloadsDir != null) {
+			File dir = new File(sharedDownloadsDir);
+			if (dir.exists() == false || dir.canWrite() == false) {
+				System.err.println("ERROR: The shared-zip directory must exist and be writeable");
+				return;
+			}
+			config.setSharedDownloadsDirectory(dir);
+		}
+
 		if (cache_dir != null) {
 			File a_dir = new File(cache_dir);
 			a_dir.mkdirs();