From 6429dac7d441ca73d207a131e40964dab3b48f2b Mon Sep 17 00:00:00 2001 From: Jim Robinson <933148+jrobinso@users.noreply.github.com> Date: Thu, 16 Jan 2025 20:24:02 -0800 Subject: [PATCH] S3 (#1639) S3 updates * Support custom, non-AWS, endpoint * Simplification of oAuth code * Remove custom S3Presigner code, this is now included in the SDK * Add AWS endpoint url preference * Allow selection of file indexes from S3 load dialog --- .../java/org/broad/igv/DirectoryManager.java | 6 +- .../java/org/broad/igv/aws/S3LoadDialog.java | 64 ++- .../broad/igv/lists/GeneListManagerUI.java | 2 +- .../org/broad/igv/oauth/OAuthProvider.java | 41 +- .../java/org/broad/igv/prefs/Constants.java | 4 +- .../org/broad/igv/prefs/IGVPreferences.java | 4 +- .../broad/igv/prefs/PreferencesEditor.java | 4 +- .../java/org/broad/igv/track/TrackLoader.java | 5 - .../java/org/broad/igv/ui/IGVMenuBar.java | 3 +- .../ui/action/ExportRegionsMenuAction.java | 2 +- .../igv/ui/action/LoadFromURLMenuAction.java | 25 +- .../broad/igv/ui/util/FileDialogUtils.java | 3 +- .../java/org/broad/igv/util/AmazonUtils.java | 464 +++++++++++------- .../java/org/broad/igv/util/HttpUtils.java | 3 +- .../java/org/broad/igv/util/S3Presigner.java | 195 -------- .../broad/igv/util/blat/BlatQueryWindow.java | 2 +- .../org/broad/igv/prefs/preferences.tab | 4 +- 17 files changed, 385 insertions(+), 446 deletions(-) delete mode 100644 src/main/java/org/broad/igv/util/S3Presigner.java diff --git a/src/main/java/org/broad/igv/DirectoryManager.java b/src/main/java/org/broad/igv/DirectoryManager.java index 994c274c4b..af84bbb30d 100644 --- a/src/main/java/org/broad/igv/DirectoryManager.java +++ b/src/main/java/org/broad/igv/DirectoryManager.java @@ -61,7 +61,7 @@ public class DirectoryManager { final public static String IGV_DIR_USERPREF = "igvDir"; - private static File getUserHome() { + public static File getUserHome() { if (USER_HOME == null) { String userHomeString = System.getProperty("user.home"); USER_HOME = new File(userHomeString); @@ -73,7 +73,7 @@ private static File getUserHome() { * The user directory. On Mac and Linux this should be the user home directory. On Windows platforms this * is the "My Documents" directory. */ - public static synchronized File getUserDirectory() { + public static synchronized File getUserDefaultDirectory() { if (USER_DIRECTORY == null) { USER_DIRECTORY = FileSystemView.getFileSystemView().getDefaultDirectory(); //Mostly for testing, in some environments USER_DIRECTORY can be null @@ -96,7 +96,7 @@ public static File getIgvDirectory() { // Hack for known Java / Windows bug. Attempt to remvoe (possible) read-only bit from user directory if (System.getProperty("os.name").startsWith("Windows")) { try { - Runtime.getRuntime().exec("attrib -r \"" + getUserDirectory().getAbsolutePath() + "\""); + Runtime.getRuntime().exec("attrib -r \"" + getUserDefaultDirectory().getAbsolutePath() + "\""); } catch (Exception e) { // We tried } diff --git a/src/main/java/org/broad/igv/aws/S3LoadDialog.java b/src/main/java/org/broad/igv/aws/S3LoadDialog.java index 5f4a6b8700..957f216799 100644 --- a/src/main/java/org/broad/igv/aws/S3LoadDialog.java +++ b/src/main/java/org/broad/igv/aws/S3LoadDialog.java @@ -44,8 +44,7 @@ import java.awt.event.ActionEvent; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; -import java.util.ArrayList; -import java.util.Collections; +import java.util.*; import java.util.List; import software.amazon.awssdk.services.s3.model.S3Exception; @@ -57,6 +56,8 @@ public class S3LoadDialog extends org.broad.igv.ui.IGVDialog { private static Logger log = LogManager.getLogger(S3LoadDialog.class); + final static Set indexExtensions = Set.of("bai", "csi", "crai", "idx", "tbi"); + private final DefaultTreeModel treeModel; String selectedId; @@ -102,7 +103,6 @@ private void loadSelections(TreePath[] paths) { LongRunningTask.submit(() -> { ArrayList> preLocatorPaths = new ArrayList<>(); - ArrayList finalLocators = new ArrayList<>(); for (TreePath path : paths) { if (isFilePath(path)) { @@ -118,15 +118,12 @@ private void loadSelections(TreePath[] paths) { } } - for (Triple preLocator : preLocatorPaths) { - ResourceLocator locator = getResourceLocatorFromBucketKey(preLocator); - finalLocators.add(locator); - } + List locators = getResourceLocatorsFromBucketKeys(preLocatorPaths); - if (finalLocators.size() == 1 && "xml".equals(ResourceLocator.deriveFormat(finalLocators.get(0).getPath()))) { - IGV.getInstance().loadSession(finalLocators.get(0).getPath(), null); + if (locators.size() == 1 && "xml".equals(ResourceLocator.deriveFormat(locators.get(0).getPath()))) { + IGV.getInstance().loadSession(locators.get(0).getPath(), null); } else { - IGV.getInstance().loadTracks(finalLocators); + IGV.getInstance().loadTracks(locators); } }); } @@ -135,11 +132,48 @@ private boolean isFilePath(TreePath path) { return ((S3TreeNode) path.getLastPathComponent()).isLeaf(); } - private ResourceLocator getResourceLocatorFromBucketKey(Triple preLocator) { - String bucketName = preLocator.getLeft(); - String s3objPath = preLocator.getMiddle(); - String s3Path = "s3://" + bucketName + "/" + s3objPath; - return new ResourceLocator(s3Path); + private List getResourceLocatorsFromBucketKeys(List> bucketKeys) { + + Map indexMap = new HashMap<>(); + List locators = new ArrayList<>(); + for (Triple bucketKey : bucketKeys) { + + String s3Path = "s3://" + bucketKey.getLeft() + "/" + bucketKey.getMiddle(); + + int idx = s3Path.lastIndexOf('.'); + String ext = idx > 0 && idx < s3Path.length() - 1 ? s3Path.substring(idx + 1) : ""; + + if (indexExtensions.contains(ext)) { + String key = s3Path.substring(0, idx); + indexMap.put(key, s3Path); + } else { + locators.add(new ResourceLocator(s3Path)); + } + + if (indexMap.size() > 0) { + for (ResourceLocator locator : locators) { + String key = locator.getPath(); + if (indexMap.containsKey(key)) { + locator.setIndexPath(indexMap.get(key)); + } else if (key.endsWith(".bam")) { + + // Special case for "Picard" which uses a non standard index naming convention + key = key.substring(0, key.length() - 4); + if (indexMap.containsKey(key)) { + locator.setIndexPath(indexMap.get(key)); + } + } else if (key.endsWith(".cram")) { + + // Special case for "Picard" which uses a non standard index naming convention + key = key.substring(0, key.length() - 5); + if (indexMap.containsKey(key)) { + locator.setIndexPath(indexMap.get(key)); + } + } + } + } + } + return locators; } private Triple getBucketKeyTierFromTreePath(TreePath path) { diff --git a/src/main/java/org/broad/igv/lists/GeneListManagerUI.java b/src/main/java/org/broad/igv/lists/GeneListManagerUI.java index da1c21e7ef..501a891077 100644 --- a/src/main/java/org/broad/igv/lists/GeneListManagerUI.java +++ b/src/main/java/org/broad/igv/lists/GeneListManagerUI.java @@ -338,7 +338,7 @@ private void importButtonActionPerformed(ActionEvent e) { */ private void exportButtonActionPerformed(ActionEvent e) { if (selectedGroup != null) { - File userDir = DirectoryManager.getUserDirectory(); + File userDir = DirectoryManager.getUserDefaultDirectory(); File initFile = new File(selectedGroup + ".gmt"); File glFile = FileDialogUtils.chooseFile("Save gene lists", userDir, initFile, FileDialogUtils.SAVE); if (glFile != null) { diff --git a/src/main/java/org/broad/igv/oauth/OAuthProvider.java b/src/main/java/org/broad/igv/oauth/OAuthProvider.java index 9e0734ee5a..3bdda7f003 100644 --- a/src/main/java/org/broad/igv/oauth/OAuthProvider.java +++ b/src/main/java/org/broad/igv/oauth/OAuthProvider.java @@ -230,16 +230,6 @@ public void fetchAccessToken(String authorizationCode) throws IOException { fetchUserProfile(payload); } - if (authProvider != null && "Amazon".equals(authProvider)) { - // Get AWS credentials after getting relevant tokens - Credentials aws_credentials; - aws_credentials = AmazonUtils.GetCognitoAWSCredentials(); - - // Update S3 client with newly acquired token - AmazonUtils.updateS3Client(aws_credentials); - } - - // Notify UI that we are authz'd/authn'd if (isLoggedIn()) { IGVEventBus.getInstance().post(new AuthStateEvent(true, this.authProvider, this.getCurrentUserName())); @@ -247,7 +237,6 @@ public void fetchAccessToken(String authorizationCode) throws IOException { } catch (Exception e) { log.error(e); - e.printStackTrace(); } } @@ -256,7 +245,7 @@ public void setAccessToken(String accessToken) { } /** - * Fetch a new access token from a refresh token. + * Fetch a new access token from a refresh token. Unlike authorization, this is a synchronous operation * * @throws IOException */ @@ -293,18 +282,15 @@ private void refreshAccessToken() throws IOException { expirationTime = System.currentTimeMillis() + response.getAsJsonPrimitive("expires_in").getAsInt() * 1000; } else { // Refresh token has failed, reauthorize from scratch - reauthorize(); + logout(); + try { + openAuthorizationPage(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } } } - private void reauthorize() throws IOException { - logout(); - try { - openAuthorizationPage(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - } /** * Extract user information from the claim information @@ -374,6 +360,15 @@ public void logout() { IGVEventBus.getInstance().post(new AuthStateEvent(false, this.authProvider, null)); } + public JsonObject getAuthorizationResponse() { + + if (response == null) { + // Go back to auth flow, not auth'd yet + checkLogin(); + response = getResponse(); + } + return response; + } /** * If not logged in, attempt to login @@ -390,10 +385,10 @@ public synchronized void checkLogin() { } } - // wait until authentication successful or 1 minute - + // wait until authentication successful or 2 minutes - // dwm08 int i = 0; - while (!isLoggedIn() && i < 600) { + while (!isLoggedIn() && i < 1200) { ++i; try { Thread.sleep(100); diff --git a/src/main/java/org/broad/igv/prefs/Constants.java b/src/main/java/org/broad/igv/prefs/Constants.java index 28f12ead80..22463a29f0 100644 --- a/src/main/java/org/broad/igv/prefs/Constants.java +++ b/src/main/java/org/broad/igv/prefs/Constants.java @@ -296,10 +296,12 @@ private Constants() { public static final String DB_NAME = "DB_NAME"; public static final String DB_PORT = "DB_PORT"; - // OAuth provisioning + // OAuth and AWS public static final String PROVISIONING_URL = "PROVISIONING.URL"; public static final String PROVISIONING_URL_DEFAULT = "PROVISIONING_URL_DEFAULT"; + public static final String AWS_ENDPOINT_URL = "AWS_ENDPOINT_URL"; + // JBrowse circular view integration public static final String CIRC_VIEW_ENABLED = "CIRC_VIEW_ENABLED"; public static final String CIRC_VIEW_PORT = "CIRC_VIEW_PORT"; diff --git a/src/main/java/org/broad/igv/prefs/IGVPreferences.java b/src/main/java/org/broad/igv/prefs/IGVPreferences.java index 497b9e2ca6..9118c1afd5 100644 --- a/src/main/java/org/broad/igv/prefs/IGVPreferences.java +++ b/src/main/java/org/broad/igv/prefs/IGVPreferences.java @@ -526,7 +526,7 @@ public File getDefineGenomeInputDirectory() { File directory = null; - String lastFilePath = get(DEFINE_GENOME_INPUT_DIRECTORY_KEY, DirectoryManager.getUserDirectory().getAbsolutePath()); + String lastFilePath = get(DEFINE_GENOME_INPUT_DIRECTORY_KEY, DirectoryManager.getUserDefaultDirectory().getAbsolutePath()); if (lastFilePath != null) { directory = new File(lastFilePath); @@ -550,7 +550,7 @@ public File getLastGenomeImportDirectory() { File genomeImportDirectory = null; - String lastFilePath = get(LAST_GENOME_IMPORT_DIRECTORY, DirectoryManager.getUserDirectory().getAbsolutePath()); + String lastFilePath = get(LAST_GENOME_IMPORT_DIRECTORY, DirectoryManager.getUserDefaultDirectory().getAbsolutePath()); if (lastFilePath != null) { genomeImportDirectory = new File(lastFilePath); diff --git a/src/main/java/org/broad/igv/prefs/PreferencesEditor.java b/src/main/java/org/broad/igv/prefs/PreferencesEditor.java index fc30673000..4224aafb12 100644 --- a/src/main/java/org/broad/igv/prefs/PreferencesEditor.java +++ b/src/main/java/org/broad/igv/prefs/PreferencesEditor.java @@ -289,7 +289,7 @@ public void focusLost(FocusEvent e) { moveButton.addActionListener(event -> { UIUtilities.invokeOnEventThread(() -> { final File directory = DirectoryManager.getFastaCacheDirectory(); - final File newDirectory = FileDialogUtils.chooseDirectory("Select cache directory", DirectoryManager.getUserDirectory()); + final File newDirectory = FileDialogUtils.chooseDirectory("Select cache directory", DirectoryManager.getUserDefaultDirectory()); if (newDirectory != null && !newDirectory.equals(directory)) { DirectoryManager.moveDirectoryContents(directory, newDirectory); SwingUtilities.invokeLater(() -> currentDirectoryLabel.setText(newDirectory.getAbsolutePath())); @@ -320,7 +320,7 @@ public void focusLost(FocusEvent e) { moveButton.addActionListener(event -> { UIUtilities.invokeOnEventThread(() -> { final File igvDirectory = DirectoryManager.getIgvDirectory(); - final File newDirectory = FileDialogUtils.chooseDirectory("Select IGV directory", DirectoryManager.getUserDirectory()); + final File newDirectory = FileDialogUtils.chooseDirectory("Select IGV directory", DirectoryManager.getUserDefaultDirectory()); if (newDirectory != null && !newDirectory.equals(igvDirectory)) { DirectoryManager.moveIGVDirectory(newDirectory); SwingUtilities.invokeLater(() -> currentDirectoryLabel.setText(newDirectory.getAbsolutePath())); diff --git a/src/main/java/org/broad/igv/track/TrackLoader.java b/src/main/java/org/broad/igv/track/TrackLoader.java index ad43440064..3637a9997e 100644 --- a/src/main/java/org/broad/igv/track/TrackLoader.java +++ b/src/main/java/org/broad/igv/track/TrackLoader.java @@ -120,11 +120,6 @@ public List load(ResourceLocator locator, Genome genome) throws DataLoadE final String path = locator.getPath().trim(); - // Check if the AWS credentials are still valid. If not, re-login and renew pre-signed urls - if (AmazonUtils.isAwsS3Path(path)) { - AmazonUtils.checkLogin(); - } - log.info("Loading resource: " + (locator.isDataURL() ? "" : path)); try { diff --git a/src/main/java/org/broad/igv/ui/IGVMenuBar.java b/src/main/java/org/broad/igv/ui/IGVMenuBar.java index bb59c34f06..6ec97373f7 100644 --- a/src/main/java/org/broad/igv/ui/IGVMenuBar.java +++ b/src/main/java/org/broad/igv/ui/IGVMenuBar.java @@ -67,7 +67,6 @@ import javax.swing.*; import javax.swing.event.MenuEvent; -import javax.swing.event.MenuListener; import javax.swing.plaf.basic.BasicBorders; import java.awt.*; import java.awt.event.ActionEvent; @@ -474,7 +473,7 @@ private JMenu createGenomesMenu() { try { File importDirectory = PreferencesManager.getPreferences().getLastGenomeImportDirectory(); if (importDirectory == null) { - PreferencesManager.getPreferences().setLastGenomeImportDirectory(DirectoryManager.getUserDirectory()); + PreferencesManager.getPreferences().setLastGenomeImportDirectory(DirectoryManager.getUserDefaultDirectory()); } // Display the dialog File file = FileDialogUtils.chooseFile("Load Genome", importDirectory, FileDialog.LOAD); diff --git a/src/main/java/org/broad/igv/ui/action/ExportRegionsMenuAction.java b/src/main/java/org/broad/igv/ui/action/ExportRegionsMenuAction.java index acaa9cb540..a3effe511e 100644 --- a/src/main/java/org/broad/igv/ui/action/ExportRegionsMenuAction.java +++ b/src/main/java/org/broad/igv/ui/action/ExportRegionsMenuAction.java @@ -78,7 +78,7 @@ public final void exportRegionsOfInterest() { File exportRegionDirectory = PreferencesManager.getPreferences().getLastExportedRegionDirectory(); if (exportRegionDirectory == null) { - exportRegionDirectory = DirectoryManager.getUserDirectory(); + exportRegionDirectory = DirectoryManager.getUserDefaultDirectory(); } String title = "Export Regions of Interest ..."; diff --git a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java index 12823f77a4..d464afa991 100644 --- a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java +++ b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java @@ -109,9 +109,7 @@ private void loadUrls(List inputs, List indexes, boolean isHtsGe } else if (inputs.size() == 1 && SessionReader.isSessionFile(inputs.getFirst())) { // Session URL String url = inputs.getFirst(); - if (url.startsWith("s3://")) { - checkAWSAccessbility(url); - } + try { LongRunningTask.submit(() -> this.igv.loadSession(url, null)); } catch (Exception ex) { @@ -186,29 +184,12 @@ private static boolean isHubURL(String input) { private static void checkURLs(List urls) { for (String url : urls) { - if (url.startsWith("s3://")) { - checkAWSAccessbility(url); - } else if (url.startsWith("ftp://")) { + if (url.startsWith("ftp://")) { MessageUtils.showMessage("FTP protocol is not supported"); } } } - private static void checkAWSAccessbility(String url) { - try { - // If AWS support is active, check if objects are in accessible tiers via Load URL menu... - if (AmazonUtils.isAwsS3Path(url)) { - String bucket = AmazonUtils.getBucketFromS3URL(url); - String key = AmazonUtils.getKeyFromS3URL(url); - AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucket, key); - if (!res.isObjectAvailable()) { - MessageUtils.showErrorMessage(res.getErrorReason(), null); - } - } - } catch (NullPointerException npe) { - // User has not yet done Amazon->Login sequence - AmazonUtils.checkLogin(); - } - } + } diff --git a/src/main/java/org/broad/igv/ui/util/FileDialogUtils.java b/src/main/java/org/broad/igv/ui/util/FileDialogUtils.java index d7cb1b24e6..cf840252eb 100644 --- a/src/main/java/org/broad/igv/ui/util/FileDialogUtils.java +++ b/src/main/java/org/broad/igv/ui/util/FileDialogUtils.java @@ -33,7 +33,6 @@ import javax.swing.*; import java.awt.*; import java.io.File; -import java.io.FileFilter; import java.io.FilenameFilter; import java.lang.reflect.Method; @@ -54,7 +53,7 @@ public static File chooseFile(String title, File initialDirectory, int mode) { } public static File chooseFile(String title) { - return chooseFile(title, DirectoryManager.getUserDirectory(), null, FileDialog.LOAD); + return chooseFile(title, DirectoryManager.getUserDefaultDirectory(), null, FileDialog.LOAD); } public static File chooseFile(String title, File initialDirectory, File initialFile, int mode) { diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index e832e5a67d..94a374124e 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -1,17 +1,16 @@ package org.broad.igv.util; import com.google.gson.JsonObject; -import org.broad.igv.Globals; +import org.broad.igv.DirectoryManager; import org.broad.igv.aws.IGVS3Object; import org.broad.igv.oauth.OAuthProvider; import org.broad.igv.oauth.OAuthUtils; import org.broad.igv.logging.LogManager; import org.broad.igv.logging.Logger; +import org.broad.igv.prefs.Constants; +import org.broad.igv.prefs.PreferencesManager; import org.broad.igv.ui.IGVMenuBar; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.auth.credentials.*; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; import software.amazon.awssdk.regions.Region; @@ -23,19 +22,24 @@ import software.amazon.awssdk.services.cognitoidentity.model.GetOpenIdTokenRequest; import software.amazon.awssdk.services.cognitoidentity.model.GetOpenIdTokenResponse; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityResponse; import software.amazon.awssdk.services.sts.model.Credentials; -import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Duration; import java.time.Instant; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -50,24 +54,15 @@ public class AmazonUtils { private static CognitoIdentityClient cognitoIdentityClient; private static Region AWSREGION; private static Boolean awsCredentialsPresent = null; - private static Credentials cognitoAWSCredentials = null; - private static int TOKEN_EXPIRE_GRACE_TIME = 1000 * 60; // 1 minute - - - - /** - * Maps s3:// URLs to presigned URLs - */ private static Map s3ToPresignedMap = new HashMap<>(); - /** - * Maps aws presigned URLs to s3://. This is needed in some cases (e.g. Tribble) to regenerate an expired URL - */ private static Map presignedToS3Map = new HashMap<>(); private static JsonObject CognitoConfig; + private static S3Presigner s3Presigner; + private static String endpointURL = "UNKNOWN"; public static void setCognitoConfig(JsonObject json) { CognitoConfig = json; @@ -97,7 +92,7 @@ public static boolean isAwsProviderPresent() { log.info("AWS configuration found. AWS support enabled."); awsCredentialsPresent = true; } else { - log.info("AWS configuration not found."); + log.info("Cognito configuration found but Amazon auth_provider not defined. Only Amazon provider is supported at this time."); awsCredentialsPresent = false; } } catch (NullPointerException np) { @@ -117,6 +112,11 @@ public static boolean isAwsProviderPresent() { return awsCredentialsPresent; } + /** + * Return the region for AWS credentials + * + * @return + */ private static Region getAWSREGION() { if (AWSREGION == null) { @@ -126,7 +126,12 @@ private static Region getAWSREGION() { // TODO -- find region in default place try { AWSREGION = (new DefaultAwsRegionProviderChain()).getRegion(); + if (AWSREGION == null) { + AWSREGION = Region.US_EAST_1; + log.info("Could not find AWS region setting. Assuming us-east-1"); + } } catch (Exception e) { + log.info("Unable to load region from any of the providers in the chain software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain. Assuming us-east-1"); AWSREGION = Region.US_EAST_1; } } @@ -135,7 +140,7 @@ private static Region getAWSREGION() { } /** - * Returns the AWS credentials + * Retrieve AWS credentials, which may trigger a login. * * @return returns the credentials based on the AWS STS access token returned from the AWS Cognito user pool. */ @@ -145,85 +150,73 @@ public static Credentials GetCognitoAWSCredentials() { log.debug("fetch credentials"); OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); - JsonObject igv_oauth_conf = GetCognitoConfig(); - JsonObject response = provider.getResponse(); - - // Handle non-user initiated S3 auth (IGV early startup), i.e user-specified GenomesLoader - if (response == null) { - // Go back to auth flow, not auth'd yet - checkLogin(); - response = provider.getResponse(); - } + JsonObject response = provider.getAuthorizationResponse(); - JsonObject payload = JWTParser.getPayload(response.get("id_token").getAsString()); - - log.debug("JWT payload id token: " + payload); - - // Collect necessary information from federated IdP for Authentication purposes - String idTokenStr = response.get("id_token").getAsString(); - String idProvider = payload.get("iss").toString().replace("https://", "") - .replace("\"", ""); - String email = payload.get("email").getAsString(); - String federatedPoolId = igv_oauth_conf.get("aws_cognito_fed_pool_id").getAsString(); - String cognitoRoleARN = igv_oauth_conf.get("aws_cognito_role_arn").getAsString(); - - HashMap logins = new HashMap<>(); - logins.put(idProvider, idTokenStr); - - // Avoid "software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(" - // The use of the AnonymousCredentialsProvider essentially bypasses the provider chain's requirement to access ~/.aws/credentials. - // https://stackoverflow.com/questions/36604024/sts-saml-and-java-sdk-unable-to-load-aws-credentials-from-any-provider-in-the-c - AnonymousCredentialsProvider anoCredProv = AnonymousCredentialsProvider.create(); - - // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cognitoidentity/CognitoIdentityClient.html - // Build the Cognito client - CognitoIdentityClientBuilder cognitoIdentityBuilder = CognitoIdentityClient.builder(); - - cognitoIdentityBuilder.region(getAWSREGION()).credentialsProvider(anoCredProv); - cognitoIdentityClient = cognitoIdentityBuilder.build(); - - - // https://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html - // Basic (Classic) Authflow - // Uses AssumeRoleWithWebIdentity and facilitates CloudTrail org.broad.igv.logging. Uses one more request but provides user traceability. - GetIdRequest.Builder idrequest = GetIdRequest.builder().identityPoolId(federatedPoolId) - .logins(logins); - GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); - - GetOpenIdTokenRequest.Builder openidrequest = GetOpenIdTokenRequest.builder().logins(logins).identityId(idResult.identityId()); - GetOpenIdTokenResponse openId = cognitoIdentityClient.getOpenIdToken(openidrequest.build()); - - - AssumeRoleWithWebIdentityRequest.Builder webidrequest = AssumeRoleWithWebIdentityRequest.builder().webIdentityToken(openId.token()) - .roleSessionName(email) - .roleArn(cognitoRoleARN); - - AssumeRoleWithWebIdentityResponse stsClientResponse = StsClient.builder().credentialsProvider(anoCredProv) - .region(getAWSREGION()) - .build() - .assumeRoleWithWebIdentity(webidrequest.build()); - -// // Enhanced (Simplified) Authflow -// // Major drawback: Does not store federated user information on CloudTrail only authenticated role name appears in logs. -// -// // "To provide end-user credentials, first make an unsigned call to GetId." -// GetIdRequest.Builder idrequest = GetIdRequest.builder().identityPoolId(federatedPoolId) -// .logins(logins); -// GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); -// -// // "Next, make an unsigned call to GetCredentialsForIdentity." -// GetCredentialsForIdentityRequest.Builder authedIds = GetCredentialsForIdentityRequest.builder(); -// authedIds.identityId(idResult.identityId()).logins(logins); -// -// GetCredentialsForIdentityResponse authedRes = cognitoIdentityClient.getCredentialsForIdentity(authedIds.build()); -// -// return authedRes.credentials() - - cognitoAWSCredentials = stsClientResponse.credentials(); + setCredentialsFromOauthResponse(response); } return cognitoAWSCredentials; } + private static void setCredentialsFromOauthResponse(JsonObject response) { + + JsonObject igv_oauth_conf = GetCognitoConfig(); + + JsonObject payload = JWTParser.getPayload(response.get("id_token").getAsString()); + + log.debug("JWT payload id token: " + payload); + + // Collect necessary information from federated IdP for Authentication purposes + String idTokenStr = response.get("id_token").getAsString(); + String idProvider = payload.get("iss").toString().replace("https://", "") + .replace("\"", ""); + String email = payload.get("email").getAsString(); + String federatedPoolId = igv_oauth_conf.get("aws_cognito_fed_pool_id").getAsString(); + String cognitoRoleARN = igv_oauth_conf.get("aws_cognito_role_arn").getAsString(); + + HashMap logins = new HashMap<>(); + logins.put(idProvider, idTokenStr); + + // Avoid "software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(" + // The use of the AnonymousCredentialsProvider essentially bypasses the provider chain's requirement to access ~/.aws/credentials. + // https://stackoverflow.com/questions/36604024/sts-saml-and-java-sdk-unable-to-load-aws-credentials-from-any-provider-in-the-c + AnonymousCredentialsProvider anoCredProv = AnonymousCredentialsProvider.create(); + + // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cognitoidentity/CognitoIdentityClient.html + // Build the Cognito client + CognitoIdentityClientBuilder cognitoIdentityBuilder = CognitoIdentityClient.builder(); + + cognitoIdentityBuilder.region(getAWSREGION()).credentialsProvider(anoCredProv); + cognitoIdentityClient = cognitoIdentityBuilder.build(); + + + // https://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html + // Basic (Classic) Authflow + // Uses AssumeRoleWithWebIdentity and facilitates CloudTrail org.broad.igv.logging. Uses one more request but provides user traceability. + GetIdRequest.Builder idrequest = GetIdRequest.builder() + .identityPoolId(federatedPoolId) + .logins(logins); + GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); + + GetOpenIdTokenRequest.Builder openidrequest = GetOpenIdTokenRequest.builder().logins(logins).identityId(idResult.identityId()); + GetOpenIdTokenResponse openId = cognitoIdentityClient.getOpenIdToken(openidrequest.build()); + + + AssumeRoleWithWebIdentityRequest.Builder webidrequest = AssumeRoleWithWebIdentityRequest.builder() + .webIdentityToken(openId.token()) + .roleSessionName(email) + .roleArn(cognitoRoleARN); + + AssumeRoleWithWebIdentityResponse stsClientResponse = StsClient.builder() + .credentialsProvider(anoCredProv) + .region(getAWSREGION()) + .build() + .assumeRoleWithWebIdentity(webidrequest.build()); + + cognitoAWSCredentials = stsClientResponse.credentials(); + + updateS3Client(cognitoAWSCredentials); + } + /** * @param cognitoAWSCredentials * @return true if credentials are due to expire within next 10 seconds* @@ -238,11 +231,38 @@ private static boolean isExpired(Credentials cognitoAWSCredentials) { * * @param credentials AWS credentials */ - public static void updateS3Client(Credentials credentials) { + private static void updateS3Client(Credentials credentials) { + final Region region = getAWSREGION(); if (credentials == null) { - s3Client = S3Client.builder().region(region).build(); + // .aws/credentials, environment variable, or other AWS supported credential store + String endpointURL = null; + try { + endpointURL = getEndpointURL(); + } catch (IOException e) { + log.error("Error searching for endpoint url", e); + } + + if (endpointURL == null) { + s3Client = S3Client.builder().region(region).build(); + } else { + // Custom endpoint + try { + S3Configuration configuration = S3Configuration.builder() + .pathStyleAccessEnabled(true).build(); + + s3Client = S3Client.builder() + .endpointOverride(new URI(endpointURL)) + .serviceConfiguration(configuration) + .region(getAWSREGION()) // this is not used, but the AWS SDK requires it + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } else { + // Cognito AwsSessionCredentials creds = AwsSessionCredentials.create( credentials.accessKeyId(), credentials.secretAccessKey(), @@ -254,6 +274,19 @@ public static void updateS3Client(Credentials credentials) { } } + private static S3Client getS3Client() { + if(s3Client == null) { + if (GetCognitoConfig() != null) { + // OAuthUtils.getInstance().getAWSProvider().getAccessToken(); + updateS3Client(GetCognitoAWSCredentials()); + } else { + updateS3Client(null); + } + } + + return s3Client; + } + /** * This method returns the details of the user and bucket lists. @@ -261,22 +294,15 @@ public static void updateS3Client(Credentials credentials) { * @return bucket list */ public static List ListBucketsForUser() { + if (bucketsFinalList.isEmpty()) { - if (GetCognitoConfig() != null) { - OAuthUtils.getInstance().getAWSProvider().getAccessToken(); - updateS3Client(GetCognitoAWSCredentials()); - } else { - updateS3Client(null); - } - List bucketsList = new ArrayList<>(); + S3Client s3Client = getS3Client(); - ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); - ListBucketsResponse listBucketsResponse = s3Client.listBuckets(listBucketsRequest); - listBucketsResponse.buckets().stream().forEach(x -> bucketsList.add(x.name())); + List bucketsList = s3Client.listBuckets().buckets().stream().map(b -> b.name()).collect(Collectors.toList()); // Filter out buckets that the user does not have permissions for - bucketsFinalList = getReadableBuckets(bucketsList); + bucketsFinalList = bucketsList; //getReadableBuckets(bucketsList); } return bucketsFinalList; @@ -421,13 +447,10 @@ private static List getReadableBuckets(List buckets) { */ public static ArrayList ListBucketObjects(String bucketName, String prefix) { + ArrayList objects = new ArrayList<>(); - if (GetCognitoConfig() != null) { - OAuthUtils.getInstance().getAWSProvider().getAccessToken(); - updateS3Client(GetCognitoAWSCredentials()); - } else { - updateS3Client(null); - } + + S3Client s3Client = getS3Client(); try { // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html @@ -437,7 +460,9 @@ public static ArrayList ListBucketObjects(String bucketName, String // hierarchy using key name prefixes and delimiters as the Amazon S3 console does. The Amazon // S3 console supports a concept of folders. // """ - ListObjectsV2Request listReq = ListObjectsV2Request.builder().bucket(bucketName) + + ListObjectsV2Request listReq = ListObjectsV2Request.builder() + .bucket(bucketName) .prefix(prefix) .delimiter("/") .build(); @@ -484,41 +509,158 @@ public static String getKeyFromS3URL(String s3URL) { return s3URI.getKey(); } - // Amazon S3 Presign URLs - // Also keeps an internal mapping between ResourceLocator and active/valid signed URLs. + /** + * Create a presigned URL for the s3:// + * + * @param s3Path + * @return + * @throws IOException + */ + private static String createPresignedURL(String s3Path) throws IOException { - // TODO: Ideally the presigned URL should be generated without any of the Cognito being involved first? - // Make sure access token are valid (refreshes token internally) - S3Presigner s3Presigner; + s3Presigner = getPresigner(); - if (GetCognitoConfig() != null) { - OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); - provider.getAccessToken(); + AmazonS3URI s3URI = new AmazonS3URI(s3Path); + String bucket = s3URI.getBucket(); + String key = s3URI.getKey(); - Credentials credentials = GetCognitoAWSCredentials(); - AwsSessionCredentials creds = AwsSessionCredentials.create(credentials.accessKeyId(), - credentials.secretAccessKey(), - credentials.sessionToken()); - StaticCredentialsProvider awsCredsProvider = StaticCredentialsProvider.create(creds); + // URI presigned = s3Presigner.presignS3DownloadLink(bucket, key); + GetObjectRequest s3GetRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + GetObjectPresignRequest getObjectPresignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(60)) + .getObjectRequest(s3GetRequest) + .build(); - s3Presigner = S3Presigner.builder() - .expiration(provider.getExpirationTime()) // Duration.ofSeconds(30) // <= for testing - .awsCredentials(awsCredsProvider) - .region(getAWSREGION()) - .build(); - } else { - s3Presigner = S3Presigner.builder().build(); - } + PresignedGetObjectRequest presignedGetObjectRequest = + s3Presigner.presignGetObject(getObjectPresignRequest); - String bucket = getBucketFromS3URL(s3Path); - String key = getKeyFromS3URL(s3Path); + URL presigned = presignedGetObjectRequest.url(); - URI presigned = s3Presigner.presignS3DownloadLink(bucket, key); log.debug("AWS presigned URL from translateAmazonCloudURL is: " + presigned); return presigned.toString(); } + private static S3Presigner getPresigner() throws IOException { + + if (s3Presigner == null) { + + if (GetCognitoConfig() != null) { + OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); + provider.getAccessToken(); + + Credentials credentials = GetCognitoAWSCredentials(); + AwsSessionCredentials creds = AwsSessionCredentials.create(credentials.accessKeyId(), + credentials.secretAccessKey(), + credentials.sessionToken()); + StaticCredentialsProvider awsCredsProvider = StaticCredentialsProvider.create(creds); + + s3Presigner = S3Presigner.builder() + .credentialsProvider(awsCredsProvider) + .region(getAWSREGION()) + .build(); + } else { + final String endpointURL = getEndpointURL(); + + if (endpointURL == null) { + s3Presigner = S3Presigner.builder().build(); + } else { + // Override presigned url style -- some (all?) 3rd party S3 providers do not support virtual host style + S3Configuration configuration = S3Configuration.builder() + .pathStyleAccessEnabled(true).build(); + try { + s3Presigner = S3Presigner.builder() + .serviceConfiguration(configuration) + .endpointOverride(new URI(endpointURL)) + .region(getAWSREGION()) + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + } + return s3Presigner; + } + + /** + * Return a custom endpoint URL, if defined. The endpointURL is searched for once per session and not updated. + * Search in the follow locations. + *

+ * (1) oauth config file + * (2) igv preference + * (3) environment variable AWS_ENDPOINT_URL + * (4) aws credentials file + * (5) aws config file + * + * @return Custom AWS endpoint url or null + * @throws IOException + */ + private static String getEndpointURL() throws IOException { + + if ("UNKNOWN".equals(endpointURL)) { + + // IGV preference + endpointURL = PreferencesManager.getPreferences().get(Constants.AWS_ENDPOINT_URL); + if (endpointURL != null) { + return endpointURL; + } + + // environment variable + endpointURL = System.getenv("AWS_ENDPOINT_URL"); + if (endpointURL != null) { + return endpointURL; + } + + // oauth config + if (GetCognitoConfig() != null && GetCognitoConfig().has("endpoint_url")) { + endpointURL = GetCognitoConfig().get("endpoint_url").getAsString(); + if (endpointURL != null) { + return endpointURL; + } + } + + // Search aws directory + File awsDirectory = new File(DirectoryManager.getUserHome(), ".aws"); + if (awsDirectory.exists()) { + File credfile = new File(awsDirectory, "credentials"); + if (credfile.exists()) { + BufferedReader br = new BufferedReader(new FileReader(credfile)); + String nextLine; + while ((nextLine = br.readLine()) != null) { + String[] tokens = nextLine.split("="); + if (tokens.length == 2) { + String key = tokens[0].trim(); + if (key.equals("endpoint_url")) { + endpointURL = tokens[1].trim(); + return endpointURL; + } + } + } + } + credfile = new File(awsDirectory, "config"); + if (credfile.exists()) { + BufferedReader br = new BufferedReader(new FileReader(credfile)); + String nextLine; + while ((nextLine = br.readLine()) != null) { + String[] tokens = nextLine.split("="); + if (tokens.length == 2) { + String key = tokens[0].trim(); + if (key.equals("endpoint_url")) { + endpointURL = tokens[1].trim(); + return endpointURL; + } + } + } + } + } + } + + + return endpointURL; + } + /** * @param s3UrlString * @return @@ -540,29 +682,21 @@ public static Boolean isAwsS3Path(String path) { return (path.startsWith("s3://")); } - public static boolean isPresignedURL(String urlString) { + public static boolean isKnownPresignedURL(String urlString) { return presignedToS3Map.containsKey(urlString); } public static String updatePresignedURL(String urlString) throws IOException { String s3UrlString = presignedToS3Map.get(urlString); if (s3UrlString == null) { - throw new RuntimeException("Unrecognized presigned url: " + urlString); + // We haven't seen this url before. This shouldn't happen, but if it does we have to assume its valid + log.info("Unrecognized presigned url: " + urlString); + return urlString; } else { return translateAmazonCloudURL(s3UrlString); } } - /** - * If using Cognito, check that the use is logged in, and prompt for login if not. - */ - public static void checkLogin() { - if (GetCognitoConfig() != null && - !OAuthUtils.getInstance().getAWSProvider().isLoggedIn()) { - OAuthUtils.getInstance().getAWSProvider().checkLogin(); - } - } - /** * Checks whether a (pre)signed url is still accessible or it has expired, offline. * No extra request/head is required to the presigned object since we have all information @@ -575,34 +709,30 @@ public static void checkLogin() { **/ private static boolean isPresignedURLValid(URL url) { - boolean isValidSignedUrl; + boolean isValidSignedUrl; try { - long presignedTime = signedURLValidity(url); + Map params = StringUtils.splitQuery(url); + String amzDateStr = params.get("X-Amz-Date"); + long amzExpires = Long.parseLong(params.get("X-Amz-Expires")); + + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); // Z(ulu) -> UTC + Date amzDate = formatter.parse(amzDateStr); + + long presignedTime = amzDate.getTime() + amzExpires * 1000; + log.debug("The date of expiration is " + amzDate + ", expires after " + amzExpires + " seconds for url: " + url); + isValidSignedUrl = presignedTime - System.currentTimeMillis() - TOKEN_EXPIRE_GRACE_TIME > 0; // Duration in milliseconds } catch (ParseException e) { log.error("The AWS signed URL date parameter X-Amz-Date has incorrect formatting"); isValidSignedUrl = false; } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + log.error("Error decoding signed url", e); isValidSignedUrl = false; } return isValidSignedUrl; } - private static long signedURLValidity(URL url) throws ParseException, UnsupportedEncodingException { - Map params = StringUtils.splitQuery(url); - String amzDateStr = params.get("X-Amz-Date"); - long amzExpires = Long.parseLong(params.get("X-Amz-Expires")); - - SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); // Z(ulu) -> UTC - Date amzDate = formatter.parse(amzDateStr); - - long timeOfExpirationMillis = amzDate.getTime() + amzExpires * 1000; - - log.debug("The date of expiration is " + amzDate + ", expires after " + amzExpires + " seconds for url: " + url); - return timeOfExpirationMillis; - } } diff --git a/src/main/java/org/broad/igv/util/HttpUtils.java b/src/main/java/org/broad/igv/util/HttpUtils.java index 086d035ce0..38734e461f 100644 --- a/src/main/java/org/broad/igv/util/HttpUtils.java +++ b/src/main/java/org/broad/igv/util/HttpUtils.java @@ -715,11 +715,10 @@ private HttpURLConnection openConnection( } // If a presigned URL, check its validity and update if needed - if (AmazonUtils.isPresignedURL(url.toExternalForm())) { + if (AmazonUtils.isKnownPresignedURL(url.toExternalForm())) { url = new URL(AmazonUtils.updatePresignedURL(url.toExternalForm())); } - // If an S3 url, obtain a signed https url if (AmazonUtils.isAwsS3Path(url.toExternalForm())) { url = new URL(AmazonUtils.translateAmazonCloudURL(url.toExternalForm())); diff --git a/src/main/java/org/broad/igv/util/S3Presigner.java b/src/main/java/org/broad/igv/util/S3Presigner.java deleted file mode 100644 index cd50fe489d..0000000000 --- a/src/main/java/org/broad/igv/util/S3Presigner.java +++ /dev/null @@ -1,195 +0,0 @@ -package org.broad.igv.util; - -/* -* -* This is a transitional class until the official java-aws-sdk-v2 includes a S3 URL presigners class, see: -* -* https://github.com/aws/aws-sdk-java-v2/issues/849#issuecomment-468892839 -* https://github.com/aws/aws-sdk-java-v2/issues/203 -* -*/ - -import java.io.ByteArrayInputStream; -import java.net.URI; -import java.time.Duration; -import java.time.Instant; - -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.signer.AwsS3V4Signer; -import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.interceptor.Context.BeforeTransmission; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; -import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.http.AbortableInputStream; -import software.amazon.awssdk.http.ExecutableHttpRequest; -import software.amazon.awssdk.http.HttpExecuteRequest; -import software.amazon.awssdk.http.HttpExecuteResponse; -import software.amazon.awssdk.http.SdkHttpClient; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; - -public class S3Presigner { - - private Region region; - private AwsCredentialsProvider awsCredentialsProvider; - private Duration expirationTime; - private Integer timeOffset; - private S3PresignExecutionInterceptor presignInterceptor; - private S3Client s3Client; - - private S3Presigner() { - presignInterceptor = new S3PresignExecutionInterceptor(); - } - - public static Builder builder() { - return new Builder(); - } - - public URI presignS3DownloadLink(String bucketName, String fileName) throws SdkClientException { - try { - - GetObjectRequest s3GetRequest = GetObjectRequest.builder().bucket(bucketName).key(fileName).build(); - ResponseInputStream response = s3Client.getObject(s3GetRequest); - response.close(); - - return presignInterceptor.getSignedURI(); - } catch (Throwable t) { - if (t instanceof SdkClientException) { - throw (SdkClientException) t; - } - throw SdkClientException.builder().cause(t).build(); - } - } - - public URI presignS3UploadLink(String bucketName, String fileName) throws SdkClientException { - try { - - PutObjectRequest s3PutRequest = PutObjectRequest.builder().bucket(bucketName).key(fileName).build(); - PutObjectResponse response = s3Client.putObject(s3PutRequest, RequestBody.empty()); - - return presignInterceptor.getSignedURI(); - } catch (Throwable t) { - if (t instanceof SdkClientException) { - throw (SdkClientException) t; - } - throw SdkClientException.builder().cause(t).build(); - } - } - - public static class Builder { - S3Presigner presigner = new S3Presigner(); - - public S3Presigner build() { - if (presigner.awsCredentialsProvider == null) { - DefaultCredentialsProvider provider = DefaultCredentialsProvider.create(); - presigner.awsCredentialsProvider = provider; - } - - if (presigner.region == null) { - presigner.region = new DefaultAwsRegionProviderChain().getRegion(); - } - - if (presigner.expirationTime == null) { - presigner.expirationTime = Duration.ofDays(4); - } - - if (presigner.timeOffset == null) { - presigner.timeOffset = 2; - } - - S3ClientBuilder s3Builder = S3Client.builder().region(presigner.region).credentialsProvider(presigner.awsCredentialsProvider); - s3Builder.overrideConfiguration(ClientOverrideConfiguration.builder().addExecutionInterceptor(presigner.presignInterceptor).build()); - s3Builder.httpClient(new NullSdkHttpClient()); - presigner.s3Client = s3Builder.build(); - - return presigner; - } - - public Builder awsCredentials(AwsCredentialsProvider awsCredentialsProvider) { - presigner.awsCredentialsProvider = awsCredentialsProvider; - return this; - } - - public Builder region(Region region) { - presigner.region = region; - return this; - } - - public Builder expiration(Duration expirationTime) { - presigner.expirationTime = expirationTime; - return this; - } - - public Builder timeOffset(Integer timeOffset) { - presigner.timeOffset = timeOffset; - return this; - } - - } - - public static class NullSdkHttpClient implements SdkHttpClient { - - @Override - public void close() { - - } - - @Override - public ExecutableHttpRequest prepareRequest(HttpExecuteRequest request) { - return new ExecutableHttpRequest() { - @Override - public HttpExecuteResponse call() { - return HttpExecuteResponse.builder().response(SdkHttpResponse.builder().statusCode(200).build()).responseBody(AbortableInputStream.create(new ByteArrayInputStream(new byte[0]))).build(); - } - - @Override - public void abort() { - } - }; - } - } - - public class S3PresignExecutionInterceptor implements ExecutionInterceptor { - - final private AwsS3V4Signer signer; - private URI signedURI; - - public S3PresignExecutionInterceptor() { - this.signer = AwsS3V4Signer.create(); - } - - @Override - public void beforeTransmission(BeforeTransmission context, ExecutionAttributes executionAttributes) { - // remove all headers because a Browser that downloads the shared URL will not send the exact values. X-Amz-SignedHeaders should only contain the host header. - SdkHttpFullRequest modifiedSdkRequest = (SdkHttpFullRequest) context.httpRequest().toBuilder().clearHeaders().build(); - - - executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, awsCredentialsProvider.resolveCredentials()); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.PRESIGNER_EXPIRATION, Instant.ofEpochSecond(System.currentTimeMillis()/1000).plus(expirationTime)); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME, "s3"); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.SIGNING_REGION, region); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.TIME_OFFSET, timeOffset); - SdkHttpFullRequest signedRequest = signer.presign(modifiedSdkRequest, executionAttributes);// sign(getRequest, new ExecutionAttributes()); - signedURI = signedRequest.getUri(); - } - - public URI getSignedURI() { - return signedURI; - } - - } - -} diff --git a/src/main/java/org/broad/igv/util/blat/BlatQueryWindow.java b/src/main/java/org/broad/igv/util/blat/BlatQueryWindow.java index b157a3c9f5..7c21315cf3 100644 --- a/src/main/java/org/broad/igv/util/blat/BlatQueryWindow.java +++ b/src/main/java/org/broad/igv/util/blat/BlatQueryWindow.java @@ -108,7 +108,7 @@ private void closeItemActionPerformed(ActionEvent e) { private void saveItemActionPerformed(ActionEvent e) { - File f = FileDialogUtils.chooseFile("Save BLAT results", DirectoryManager.getUserDirectory(), FileDialogUtils.SAVE); + File f = FileDialogUtils.chooseFile("Save BLAT results", DirectoryManager.getUserDefaultDirectory(), FileDialogUtils.SAVE); if (f != null) { try { model.save(f); diff --git a/src/main/resources/org/broad/igv/prefs/preferences.tab b/src/main/resources/org/broad/igv/prefs/preferences.tab index 47ed001796..470ff9b7ec 100644 --- a/src/main/resources/org/broad/igv/prefs/preferences.tab +++ b/src/main/resources/org/broad/igv/prefs/preferences.tab @@ -273,9 +273,9 @@ MASTER_RESOURCE_FILE_KEY Data registry URL string https://igv.org/genomes/regist --- PROVISIONING.URL OAuth provisioning URL string null --- - -BLAT_URL BLAT URL String https://genome.ucsc.edu/cgi-bin/hgBlat +BLAT_URL BLAT URL string https://genome.ucsc.edu/cgi-bin/hgBlat --- +AWS_ENDPOINT_URL AWS endpoint URL string null ## Tooltip TOOLTIP.INITIAL_DELAY Tooltip inital delay (ms) integer 50