diff --git a/pyspedas/projects/maven/download_files_utilities.py b/pyspedas/projects/maven/download_files_utilities.py index 0e8ffc6c..b39d2014 100644 --- a/pyspedas/projects/maven/download_files_utilities.py +++ b/pyspedas/projects/maven/download_files_utilities.py @@ -42,14 +42,14 @@ def get_filenames(query, public): handler = urllib.request.HTTPBasicAuthHandler(p) opener = urllib.request.build_opener(handler) urllib.request.install_opener(opener) - logging.debug("get_filenames() making request to private URL: %s", private_url) + logging.info("get_filenames() making request to private URL: %s", private_url) page = urllib.request.urlopen(private_url) - logging.debug("get_filenames() finished request to private URL: %s", private_url) + logging.info("get_filenames() finished request to private URL: %s", private_url) else: - logging.debug("get_filenames() making request to public URL: %s", public_url) + logging.info("get_filenames() making request to public URL: %s", public_url) page = urllib.request.urlopen(public_url) - logging.debug("get_filenames() finished request to public URL: %s", public_url) + logging.info("get_filenames() finished request to public URL: %s", public_url) return page.read().decode("utf-8") @@ -160,7 +160,7 @@ def get_orbit_files(): fo = fs.open(ifn, "wb") else: ifn = os.path.join(orbit_files_path, filename) - logging.debug("reading plain file %s",ifn) + logging.info("reading plain file %s",ifn) fo = open(ifn, "wb") with fo as code: content=o_file.read() @@ -177,7 +177,7 @@ def merge_orbit_files(): Merge MAVEN orbit files into a single file. This function searches for MAVEN orbit files in the 'orbitfiles' directory and merges them into a single file - named 'maven_orb_rec.orb' in the same directory. The files are sorted based on their dates before merging. + named 'maven_orb_rec_merged.orb' in the same directory. The files are sorted based on their dates before merging. Returns: None @@ -195,12 +195,14 @@ def merge_orbit_files(): orbit_files_path = "/".join([toolkit_path, "orbitfiles"]) fl = fs.listdir(orbit_files_path, detail=False) fl = [f.rstrip("/").split("/")[-1] for f in fl] - fo = fs.open("/".join([orbit_files_path, "maven_orb_rec.orb"]), "w") + output_filename = "/".join([orbit_files_path, "maven_orb_rec_merged.orb"]) + fo = fs.open(output_filename, "w") else: orbit_files_path = os.path.join(toolkit_path, "orbitfiles") fl = os.listdir(orbit_files_path) - fo = open(os.path.join(orbit_files_path, "maven_orb_rec.orb"), "w") + output_filename = os.path.join(orbit_files_path,"maven_orb_rec_merged.org") + fo = open(output_filename, "w") pattern = "maven_orb_rec(_|)(|.{6})(|_.{9}).orb" orb_dates = [] @@ -220,6 +222,7 @@ def merge_orbit_files(): with fo as code: skip_2_lines = False for o_file in sorted_files: + logging.info("merge_orbit_files processing file %s", o_file) if is_fsspec_uri(toolkit_path): # assumes fsspec filesystem triggered above fo_file = fs.open(o_file) @@ -231,6 +234,7 @@ def merge_orbit_files(): f.readline() skip_2_lines = True content=f.read() + logging.info("writing %d bytes to output file %s",len(content),output_filename) if type(content) is bytes: code.write(str(content)) else: diff --git a/pyspedas/projects/maven/maven_load.py b/pyspedas/projects/maven/maven_load.py index 22f8d64c..24bc3ac4 100644 --- a/pyspedas/projects/maven/maven_load.py +++ b/pyspedas/projects/maven/maven_load.py @@ -140,6 +140,7 @@ def maven_filenames( query = "&".join(query_args) + logging.info("Querying for filenames: %s", query) s = get_filenames(query, public) if not s: @@ -149,6 +150,7 @@ def maven_filenames( continue s = s.split(",") + logging.info("Returned %d files",len(s)) if instrument not in maven_files: maven_files[instrument] = [s, data_dir, public] diff --git a/pyspedas/projects/maven/orbit_time.py b/pyspedas/projects/maven/orbit_time.py index 79518969..5f3ed05a 100644 --- a/pyspedas/projects/maven/orbit_time.py +++ b/pyspedas/projects/maven/orbit_time.py @@ -74,7 +74,8 @@ def orbit_time(begin_orbit, end_orbit=None): toolkit_path = CONFIG["local_data_dir"] sep = "/" if is_fsspec_uri(toolkit_path) else os.path.sep orbit_files_path = sep.join([toolkit_path, "orbitfiles"]) - orb_file = sep.join([orbit_files_path, "maven_orb_rec.orb"]) + orb_file = sep.join([orbit_files_path, "maven_orb_rec_merged.orb"]) + logging.info("Getting orbit info from file %s", orb_file) if is_fsspec_uri(toolkit_path): protocol, path = toolkit_path.split("://") fs = fsspec.filesystem(protocol) diff --git a/pyspedas/projects/maven/tests/tests.py b/pyspedas/projects/maven/tests/tests.py index a47b3d1c..41193ff5 100644 --- a/pyspedas/projects/maven/tests/tests.py +++ b/pyspedas/projects/maven/tests/tests.py @@ -37,7 +37,7 @@ def test_get_merge_orbit_files(self): get_orbit_files() merge_orbit_files() orbfilepath = os.path.join( - CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec.orb" + CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec_merged.orb" ) self.assertTrue(os.path.exists(orbfilepath)) diff --git a/pyspedas/projects/maven/tests/uri_tests.py b/pyspedas/projects/maven/tests/uri_tests.py index 6799b6dc..9bb1c442 100644 --- a/pyspedas/projects/maven/tests/uri_tests.py +++ b/pyspedas/projects/maven/tests/uri_tests.py @@ -6,7 +6,8 @@ import subprocess from pytplot import data_exists, tplot_names -from pyspedas import maven +import pyspedas +from pyspedas.projects import maven from pyspedas.projects.maven import config, \ utilities, \ maven_kp_to_tplot, \ @@ -32,6 +33,8 @@ # We need sleep time to avoid "HTTP Error 429: Too Many Requests" sleep_time = 45 +import logging + class LoadTestCases(unittest.TestCase): """ Cloud Awareness Unit Tests @@ -104,7 +107,7 @@ def test_get_merge_orbit_files(self): download_files_utilities.get_orbit_files() download_files_utilities.merge_orbit_files() orbfilepath = "/".join([ - config.CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec.orb" + config.CONFIG["local_data_dir"], "orbitfiles", "maven_orb_rec_merged.orb" ]) # assert file exists @@ -430,8 +433,12 @@ def test_load_mag_data_private(self): pass def test_load_mag_byorbit_data(self): + #config.CONFIG["local_data_dir"] = f"s3://{bucket_name}" + #saved_logging_level = pyspedas.logger.getEffectiveLevel() + #pyspedas.logger.setLevel(logging.DEBUG) data = maven.mag(trange=[500, 501], datatype="ss1s") self.assertTrue(len(tplot_names("OB_B*"))>0) + #pyspedas.logger.setLevel(saved_logging_level) time.sleep(sleep_time) def test_load_sta_data(self):