S3 (#1639)

S3 updates
* Support custom, non-AWS, endpoint
* Simplification of oAuth code
* Remove custom S3Presigner code, this is now included in the SDK
* Add AWS endpoint url preference
* Allow selection of file indexes from S3 load dialog

src/main/java/org/broad/igv/DirectoryManager.java

@@ -61,7 +61,7 @@ public class DirectoryManager {
     final public static String IGV_DIR_USERPREF = "igvDir";
 
 
-    private static File getUserHome() {
+    public static File getUserHome() {
         if (USER_HOME == null) {
             String userHomeString = System.getProperty("user.home");
             USER_HOME = new File(userHomeString);
@@ -73,7 +73,7 @@ private static File getUserHome() {
      * The user directory.  On Mac and Linux this should be the user home directory.  On Windows platforms this
      * is the "My Documents" directory.
      */
-    public static synchronized File getUserDirectory() {
+    public static synchronized File getUserDefaultDirectory() {
         if (USER_DIRECTORY == null) {
             USER_DIRECTORY = FileSystemView.getFileSystemView().getDefaultDirectory();
             //Mostly for testing, in some environments USER_DIRECTORY can be null
@@ -96,7 +96,7 @@ public static File getIgvDirectory() {
             // Hack for known Java / Windows bug.   Attempt to remvoe (possible) read-only bit from user directory
             if (System.getProperty("os.name").startsWith("Windows")) {
                 try {
-                    Runtime.getRuntime().exec("attrib -r \"" + getUserDirectory().getAbsolutePath() + "\"");
+                    Runtime.getRuntime().exec("attrib -r \"" + getUserDefaultDirectory().getAbsolutePath() + "\"");
                 } catch (Exception e) {
                     // We tried
                 }

src/main/java/org/broad/igv/aws/S3LoadDialog.java

@@ -44,8 +44,7 @@
 import java.awt.event.ActionEvent;
 import java.awt.event.MouseAdapter;
 import java.awt.event.MouseEvent;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.util.*;
 import java.util.List;
 
 import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -57,6 +56,8 @@ public class S3LoadDialog extends org.broad.igv.ui.IGVDialog {
 
     private static Logger log = LogManager.getLogger(S3LoadDialog.class);
 
+    final static Set<String> indexExtensions = Set.of("bai", "csi", "crai", "idx", "tbi");
+
     private final DefaultTreeModel treeModel;
     String selectedId;
 
@@ -102,7 +103,6 @@ private void loadSelections(TreePath[] paths) {
         LongRunningTask.submit(() -> {
 
             ArrayList<Triple<String, String, String>> preLocatorPaths = new ArrayList<>();
-            ArrayList<ResourceLocator> finalLocators = new ArrayList<>();
 
             for (TreePath path : paths) {
                 if (isFilePath(path)) {
@@ -118,15 +118,12 @@ private void loadSelections(TreePath[] paths) {
                 }
             }
 
-            for (Triple<String, String, String> preLocator : preLocatorPaths) {
-                ResourceLocator locator = getResourceLocatorFromBucketKey(preLocator);
-                finalLocators.add(locator);
-            }
+            List<ResourceLocator> locators = getResourceLocatorsFromBucketKeys(preLocatorPaths);
 
-            if (finalLocators.size() == 1 && "xml".equals(ResourceLocator.deriveFormat(finalLocators.get(0).getPath()))) {
-                IGV.getInstance().loadSession(finalLocators.get(0).getPath(), null);
+            if (locators.size() == 1 && "xml".equals(ResourceLocator.deriveFormat(locators.get(0).getPath()))) {
+                IGV.getInstance().loadSession(locators.get(0).getPath(), null);
             } else {
-                IGV.getInstance().loadTracks(finalLocators);
+                IGV.getInstance().loadTracks(locators);
             }
         });
     }
@@ -135,11 +132,48 @@ private boolean isFilePath(TreePath path) {
         return ((S3TreeNode) path.getLastPathComponent()).isLeaf();
     }
 
-    private ResourceLocator getResourceLocatorFromBucketKey(Triple<String, String, String> preLocator) {
-        String bucketName = preLocator.getLeft();
-        String s3objPath = preLocator.getMiddle();
-        String s3Path = "s3://" + bucketName + "/" + s3objPath;
-        return new ResourceLocator(s3Path);
+    private List<ResourceLocator> getResourceLocatorsFromBucketKeys(List<Triple<String, String, String>> bucketKeys) {
+
+        Map<String, String> indexMap = new HashMap<>();
+        List<ResourceLocator> locators = new ArrayList<>();
+        for (Triple<String, String, String> bucketKey : bucketKeys) {
+
+            String s3Path = "s3://" + bucketKey.getLeft() + "/" + bucketKey.getMiddle();
+
+            int idx = s3Path.lastIndexOf('.');
+            String ext = idx > 0 && idx < s3Path.length() - 1 ? s3Path.substring(idx + 1) : "";
+
+            if (indexExtensions.contains(ext)) {
+                String key = s3Path.substring(0, idx);
+                indexMap.put(key, s3Path);
+            } else {
+                locators.add(new ResourceLocator(s3Path));
+            }
+
+            if (indexMap.size() > 0) {
+                for (ResourceLocator locator : locators) {
+                    String key = locator.getPath();
+                    if (indexMap.containsKey(key)) {
+                        locator.setIndexPath(indexMap.get(key));
+                    } else if (key.endsWith(".bam")) {
+
+                        // Special case for "Picard" which uses a non standard index naming convention
+                        key = key.substring(0, key.length() - 4);
+                        if (indexMap.containsKey(key)) {
+                            locator.setIndexPath(indexMap.get(key));
+                        }
+                    } else if (key.endsWith(".cram")) {
+
+                        // Special case for "Picard" which uses a non standard index naming convention
+                        key = key.substring(0, key.length() - 5);
+                        if (indexMap.containsKey(key)) {
+                            locator.setIndexPath(indexMap.get(key));
+                        }
+                    }
+                }
+            }
+        }
+        return locators;
     }
 
     private Triple<String, String, String> getBucketKeyTierFromTreePath(TreePath path) {

src/main/java/org/broad/igv/lists/GeneListManagerUI.java

@@ -338,7 +338,7 @@ private void importButtonActionPerformed(ActionEvent e) {
      */
     private void exportButtonActionPerformed(ActionEvent e) {
         if (selectedGroup != null) {
-            File userDir = DirectoryManager.getUserDirectory();
+            File userDir = DirectoryManager.getUserDefaultDirectory();
             File initFile = new File(selectedGroup + ".gmt");
             File glFile = FileDialogUtils.chooseFile("Save gene lists", userDir, initFile, FileDialogUtils.SAVE);
             if (glFile != null) {

src/main/java/org/broad/igv/oauth/OAuthProvider.java

@@ -230,24 +230,13 @@ public void fetchAccessToken(String authorizationCode) throws IOException {
                 fetchUserProfile(payload);
             }
 
-            if (authProvider != null && "Amazon".equals(authProvider)) {
-                // Get AWS credentials after getting relevant tokens
-                Credentials aws_credentials;
-                aws_credentials = AmazonUtils.GetCognitoAWSCredentials();
-
-                // Update S3 client with newly acquired token
-                AmazonUtils.updateS3Client(aws_credentials);
-            }
-
-
             // Notify UI that we are authz'd/authn'd
             if (isLoggedIn()) {
                 IGVEventBus.getInstance().post(new AuthStateEvent(true, this.authProvider, this.getCurrentUserName()));
             }
 
         } catch (Exception e) {
             log.error(e);
-            e.printStackTrace();
         }
     }
 
@@ -256,7 +245,7 @@ public void setAccessToken(String accessToken) {
     }
 
     /**
-     * Fetch a new access token from a refresh token.
+     * Fetch a new access token from a refresh token.  Unlike authorization, this is a synchronous operation
      *
      * @throws IOException
      */
@@ -293,18 +282,15 @@ private void refreshAccessToken() throws IOException {
             expirationTime = System.currentTimeMillis() + response.getAsJsonPrimitive("expires_in").getAsInt() * 1000;
         } else {
             // Refresh token has failed, reauthorize from scratch
-            reauthorize();
+            logout();
+            try {
+                openAuthorizationPage();
+            } catch (URISyntaxException e) {
+                e.printStackTrace();
+            }
         }
     }
 
-    private void reauthorize() throws IOException {
-        logout();
-        try {
-            openAuthorizationPage();
-        } catch (URISyntaxException e) {
-            e.printStackTrace();
-        }
-    }
 
     /**
      * Extract user information from the claim information
@@ -374,6 +360,15 @@ public void logout() {
         IGVEventBus.getInstance().post(new AuthStateEvent(false, this.authProvider, null));
     }
 
+    public JsonObject getAuthorizationResponse() {
+
+        if (response == null) {
+            // Go back to auth flow, not auth'd yet
+            checkLogin();
+            response = getResponse();
+        }
+        return response;
+    }
 
     /**
      * If not logged in, attempt to login
@@ -390,10 +385,10 @@ public synchronized void checkLogin() {
             }
 
         }
-        // wait until authentication successful or 1 minute -
+        // wait until authentication successful or 2 minutes -
         // dwm08
         int i = 0;
-        while (!isLoggedIn() && i < 600) {
+        while (!isLoggedIn() && i < 1200) {
             ++i;
             try {
                 Thread.sleep(100);

src/main/java/org/broad/igv/prefs/Constants.java

@@ -296,10 +296,12 @@ private Constants() {
     public static final String DB_NAME = "DB_NAME";
     public static final String DB_PORT = "DB_PORT";
 
-    // OAuth provisioning
+    // OAuth and AWS
     public static final String PROVISIONING_URL = "PROVISIONING.URL";
     public static final String PROVISIONING_URL_DEFAULT = "PROVISIONING_URL_DEFAULT";
 
+    public static final String AWS_ENDPOINT_URL = "AWS_ENDPOINT_URL";
+
     // JBrowse circular view integration
     public static final String CIRC_VIEW_ENABLED = "CIRC_VIEW_ENABLED";
     public static final String CIRC_VIEW_PORT = "CIRC_VIEW_PORT";

src/main/java/org/broad/igv/prefs/IGVPreferences.java

@@ -526,7 +526,7 @@ public File getDefineGenomeInputDirectory() {
 
         File directory = null;
 
-        String lastFilePath = get(DEFINE_GENOME_INPUT_DIRECTORY_KEY, DirectoryManager.getUserDirectory().getAbsolutePath());
+        String lastFilePath = get(DEFINE_GENOME_INPUT_DIRECTORY_KEY, DirectoryManager.getUserDefaultDirectory().getAbsolutePath());
 
         if (lastFilePath != null) {
             directory = new File(lastFilePath);
@@ -550,7 +550,7 @@ public File getLastGenomeImportDirectory() {
 
         File genomeImportDirectory = null;
 
-        String lastFilePath = get(LAST_GENOME_IMPORT_DIRECTORY, DirectoryManager.getUserDirectory().getAbsolutePath());
+        String lastFilePath = get(LAST_GENOME_IMPORT_DIRECTORY, DirectoryManager.getUserDefaultDirectory().getAbsolutePath());
 
         if (lastFilePath != null) {
             genomeImportDirectory = new File(lastFilePath);

src/main/java/org/broad/igv/prefs/PreferencesEditor.java

@@ -289,7 +289,7 @@ public void focusLost(FocusEvent e) {
                 moveButton.addActionListener(event -> {
                     UIUtilities.invokeOnEventThread(() -> {
                         final File directory = DirectoryManager.getFastaCacheDirectory();
-                        final File newDirectory = FileDialogUtils.chooseDirectory("Select cache directory", DirectoryManager.getUserDirectory());
+                        final File newDirectory = FileDialogUtils.chooseDirectory("Select cache directory", DirectoryManager.getUserDefaultDirectory());
                         if (newDirectory != null && !newDirectory.equals(directory)) {
                             DirectoryManager.moveDirectoryContents(directory, newDirectory);
                             SwingUtilities.invokeLater(() -> currentDirectoryLabel.setText(newDirectory.getAbsolutePath()));
@@ -320,7 +320,7 @@ public void focusLost(FocusEvent e) {
                 moveButton.addActionListener(event -> {
                     UIUtilities.invokeOnEventThread(() -> {
                         final File igvDirectory = DirectoryManager.getIgvDirectory();
-                        final File newDirectory = FileDialogUtils.chooseDirectory("Select IGV directory", DirectoryManager.getUserDirectory());
+                        final File newDirectory = FileDialogUtils.chooseDirectory("Select IGV directory", DirectoryManager.getUserDefaultDirectory());
                         if (newDirectory != null && !newDirectory.equals(igvDirectory)) {
                             DirectoryManager.moveIGVDirectory(newDirectory);
                             SwingUtilities.invokeLater(() -> currentDirectoryLabel.setText(newDirectory.getAbsolutePath()));

src/main/java/org/broad/igv/track/TrackLoader.java

@@ -120,11 +120,6 @@ public List<Track> load(ResourceLocator locator, Genome genome) throws DataLoadE
 
         final String path = locator.getPath().trim();
 
-        // Check if the AWS credentials are still valid. If not, re-login and renew pre-signed urls
-        if (AmazonUtils.isAwsS3Path(path)) {
-            AmazonUtils.checkLogin();
-        }
-
         log.info("Loading resource:  " + (locator.isDataURL() ? "<data url>" : path));
         try {
 

src/main/java/org/broad/igv/ui/IGVMenuBar.java

@@ -67,7 +67,6 @@
 
 import javax.swing.*;
 import javax.swing.event.MenuEvent;
-import javax.swing.event.MenuListener;
 import javax.swing.plaf.basic.BasicBorders;
 import java.awt.*;
 import java.awt.event.ActionEvent;
@@ -474,7 +473,7 @@ private JMenu createGenomesMenu() {
             try {
                 File importDirectory = PreferencesManager.getPreferences().getLastGenomeImportDirectory();
                 if (importDirectory == null) {
-                    PreferencesManager.getPreferences().setLastGenomeImportDirectory(DirectoryManager.getUserDirectory());
+                    PreferencesManager.getPreferences().setLastGenomeImportDirectory(DirectoryManager.getUserDefaultDirectory());
                 }
                 // Display the dialog
                 File file = FileDialogUtils.chooseFile("Load Genome", importDirectory, FileDialog.LOAD);

src/main/java/org/broad/igv/ui/action/ExportRegionsMenuAction.java

@@ -78,7 +78,7 @@ public final void exportRegionsOfInterest() {
 
         File exportRegionDirectory = PreferencesManager.getPreferences().getLastExportedRegionDirectory();
         if (exportRegionDirectory == null) {
-            exportRegionDirectory = DirectoryManager.getUserDirectory();
+            exportRegionDirectory = DirectoryManager.getUserDefaultDirectory();
         }
 
         String title = "Export Regions of Interest ...";

src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java

@@ -109,9 +109,7 @@ private void loadUrls(List<String> inputs, List<String> indexes, boolean isHtsGe
         } else if (inputs.size() == 1 && SessionReader.isSessionFile(inputs.getFirst())) {
             // Session URL
             String url = inputs.getFirst();
-            if (url.startsWith("s3://")) {
-                checkAWSAccessbility(url);
-            }
+
             try {
                 LongRunningTask.submit(() -> this.igv.loadSession(url, null));
             } catch (Exception ex) {
@@ -186,29 +184,12 @@ private static boolean isHubURL(String input) {
 
     private static void checkURLs(List<String> urls) {
         for (String url : urls) {
-            if (url.startsWith("s3://")) {
-                checkAWSAccessbility(url);
-            } else if (url.startsWith("ftp://")) {
+             if (url.startsWith("ftp://")) {
                 MessageUtils.showMessage("FTP protocol is not supported");
             }
         }
     }
 
-    private static void checkAWSAccessbility(String url) {
-        try {
-            // If AWS support is active, check if objects are in accessible tiers via Load URL menu...
-            if (AmazonUtils.isAwsS3Path(url)) {
-                String bucket = AmazonUtils.getBucketFromS3URL(url);
-                String key = AmazonUtils.getKeyFromS3URL(url);
-                AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucket, key);
-                if (!res.isObjectAvailable()) {
-                    MessageUtils.showErrorMessage(res.getErrorReason(), null);
-                }
-            }
-        } catch (NullPointerException npe) {
-            // User has not yet done Amazon->Login sequence
-            AmazonUtils.checkLogin();
-        }
-    }
+
 }
 

src/main/java/org/broad/igv/ui/util/FileDialogUtils.java

@@ -33,7 +33,6 @@
 import javax.swing.*;
 import java.awt.*;
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FilenameFilter;
 import java.lang.reflect.Method;
 
@@ -54,7 +53,7 @@ public static File chooseFile(String title, File initialDirectory, int mode) {
     }
 
     public static File chooseFile(String title) {
-        return chooseFile(title, DirectoryManager.getUserDirectory(), null, FileDialog.LOAD);
+        return chooseFile(title, DirectoryManager.getUserDefaultDirectory(), null, FileDialog.LOAD);
     }
 
     public static File chooseFile(String title, File initialDirectory, File initialFile, int mode) {