Changed name of MAVEN merged orbit file to avoid overwriting input file.

Added logging for troubleshooting loading mag data by orbit

pyspedas/projects/maven/download_files_utilities.py

@@ -42,14 +42,14 @@ def get_filenames(query, public):
         handler = urllib.request.HTTPBasicAuthHandler(p)
         opener = urllib.request.build_opener(handler)
         urllib.request.install_opener(opener)
-        logging.debug("get_filenames() making request to private URL: %s", private_url)
+        logging.info("get_filenames() making request to private URL: %s", private_url)
         page = urllib.request.urlopen(private_url)
-        logging.debug("get_filenames() finished request to private URL: %s", private_url)
+        logging.info("get_filenames() finished request to private URL: %s", private_url)
 
     else:
-        logging.debug("get_filenames() making request to public URL: %s", public_url)
+        logging.info("get_filenames() making request to public URL: %s", public_url)
         page = urllib.request.urlopen(public_url)
-        logging.debug("get_filenames() finished request to public URL: %s", public_url)
+        logging.info("get_filenames() finished request to public URL: %s", public_url)
 
     return page.read().decode("utf-8")
 
@@ -160,7 +160,7 @@ def get_orbit_files():
             fo = fs.open(ifn, "wb")
         else:
             ifn = os.path.join(orbit_files_path, filename)
-            logging.debug("reading plain file %s",ifn)
+            logging.info("reading plain file %s",ifn)
             fo = open(ifn, "wb")
         with fo as code:
             content=o_file.read()
@@ -177,7 +177,7 @@ def merge_orbit_files():
     Merge MAVEN orbit files into a single file.
 
     This function searches for MAVEN orbit files in the 'orbitfiles' directory and merges them into a single file
-    named 'maven_orb_rec.orb' in the same directory. The files are sorted based on their dates before merging.
+    named 'maven_orb_rec_merged.orb' in the same directory. The files are sorted based on their dates before merging.
 
     Returns:
         None
@@ -195,12 +195,14 @@ def merge_orbit_files():
         orbit_files_path = "/".join([toolkit_path, "orbitfiles"])
         fl = fs.listdir(orbit_files_path, detail=False)
         fl = [f.rstrip("/").split("/")[-1] for f in fl]
-        fo = fs.open("/".join([orbit_files_path, "maven_orb_rec.orb"]), "w")
+        output_filename = "/".join([orbit_files_path, "maven_orb_rec_merged.orb"])
+        fo = fs.open(output_filename, "w")
     else:
         orbit_files_path = os.path.join(toolkit_path, "orbitfiles")
 
         fl = os.listdir(orbit_files_path)
-        fo = open(os.path.join(orbit_files_path, "maven_orb_rec.orb"), "w")
+        output_filename = os.path.join(orbit_files_path,"maven_orb_rec_merged.org")
+        fo = open(output_filename, "w")
 
     pattern = "maven_orb_rec(_|)(|.{6})(|_.{9}).orb"
     orb_dates = []
@@ -220,6 +222,7 @@ def merge_orbit_files():
     with fo as code:
         skip_2_lines = False
         for o_file in sorted_files:
+            logging.info("merge_orbit_files processing file %s", o_file)
             if is_fsspec_uri(toolkit_path):
                 # assumes fsspec filesystem triggered above
                 fo_file = fs.open(o_file)
@@ -231,6 +234,7 @@ def merge_orbit_files():
                     f.readline()
                 skip_2_lines = True
                 content=f.read()
+                logging.info("writing %d bytes to output file %s",len(content),output_filename)
                 if type(content) is bytes:
                     code.write(str(content))
                 else:

pyspedas/projects/maven/maven_load.py

@@ -140,6 +140,7 @@ def maven_filenames(
 
         query = "&".join(query_args)
 
+        logging.info("Querying for filenames: %s", query)
         s = get_filenames(query, public)
 
         if not s:
@@ -149,6 +150,7 @@ def maven_filenames(
             continue
 
         s = s.split(",")
+        logging.info("Returned %d files",len(s))
 
         if instrument not in maven_files:
             maven_files[instrument] = [s, data_dir, public]

pyspedas/projects/maven/orbit_time.py

@@ -74,7 +74,8 @@ def orbit_time(begin_orbit, end_orbit=None):
     toolkit_path = CONFIG["local_data_dir"]
     sep = "/" if is_fsspec_uri(toolkit_path) else os.path.sep
     orbit_files_path = sep.join([toolkit_path, "orbitfiles"])
-    orb_file = sep.join([orbit_files_path, "maven_orb_rec.orb"])
+    orb_file = sep.join([orbit_files_path, "maven_orb_rec_merged.orb"])
+    logging.info("Getting orbit info from file %s", orb_file)
     if is_fsspec_uri(toolkit_path):
         protocol, path = toolkit_path.split("://")
         fs = fsspec.filesystem(protocol)

pyspedas/projects/maven/tests/tests.py

@@ -37,7 +37,7 @@ def test_get_merge_orbit_files(self):
         get_orbit_files()
         merge_orbit_files()
         orbfilepath = os.path.join(
-            CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec.orb"
+            CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec_merged.orb"
         )
         self.assertTrue(os.path.exists(orbfilepath))
 

pyspedas/projects/maven/tests/uri_tests.py

@@ -6,7 +6,8 @@
 import subprocess
 
 from pytplot import data_exists, tplot_names
-from pyspedas import maven
+import pyspedas
+from pyspedas.projects import maven
 from pyspedas.projects.maven import config,                   \
                                     utilities,                \
                                     maven_kp_to_tplot,        \
@@ -32,6 +33,8 @@
 # We need sleep time to avoid "HTTP Error 429: Too Many Requests"
 sleep_time = 45
 
+import logging
+
 class LoadTestCases(unittest.TestCase):
     """
     Cloud Awareness Unit Tests
@@ -104,7 +107,7 @@ def test_get_merge_orbit_files(self):
         download_files_utilities.get_orbit_files()
         download_files_utilities.merge_orbit_files()
         orbfilepath = "/".join([
-            config.CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec.orb"
+            config.CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec_merged.orb"
         ])
 
         # assert file exists
@@ -430,8 +433,12 @@ def test_load_mag_data_private(self):
             pass
 
     def test_load_mag_byorbit_data(self):
+        #config.CONFIG["local_data_dir"] = f"s3://{bucket_name}"
+        #saved_logging_level = pyspedas.logger.getEffectiveLevel()
+        #pyspedas.logger.setLevel(logging.DEBUG)
         data = maven.mag(trange=[500, 501], datatype="ss1s")
         self.assertTrue(len(tplot_names("OB_B*"))>0)
+        #pyspedas.logger.setLevel(saved_logging_level)
         time.sleep(sleep_time)
 
     def test_load_sta_data(self):