From 13b690ac77f5fb407ab21053b8b7b5757c5115ba Mon Sep 17 00:00:00 2001 From: Kristin Date: Sun, 29 Mar 2020 10:08:10 -0700 Subject: [PATCH] Move messenger wac dem creation scripts from ISISDATA/base/templates/isisminer to this location --- messenger_wac_dem/POINTCLOUDTAB.FMT | 191 ++++++++ messenger_wac_dem/README | 41 ++ .../jigsaw_bundleout_sigmas_pdsfmt.conf | 137 ++++++ messenger_wac_dem/mdis_image_overlaps.conf | 350 ++++++++++++++ messenger_wac_dem/mdis_wac_matcher.conf | 428 ++++++++++++++++++ messenger_wac_dem/multi_algorithms.lis | 4 + messenger_wac_dem/single_algorithms.lis | 4 + 7 files changed, 1155 insertions(+) create mode 100644 messenger_wac_dem/POINTCLOUDTAB.FMT create mode 100644 messenger_wac_dem/README create mode 100644 messenger_wac_dem/jigsaw_bundleout_sigmas_pdsfmt.conf create mode 100644 messenger_wac_dem/mdis_image_overlaps.conf create mode 100644 messenger_wac_dem/mdis_wac_matcher.conf create mode 100644 messenger_wac_dem/multi_algorithms.lis create mode 100644 messenger_wac_dem/single_algorithms.lis diff --git a/messenger_wac_dem/POINTCLOUDTAB.FMT b/messenger_wac_dem/POINTCLOUDTAB.FMT new file mode 100644 index 0000000..40e9b0e --- /dev/null +++ b/messenger_wac_dem/POINTCLOUDTAB.FMT @@ -0,0 +1,191 @@ +OBJECT = COLUMN + COLUMN_NUMBER = 1 + NAME = POINT_ID + DATA_TYPE = CHARACTER + UNIT = NONE + START_BYTE = 2 + BYTES = 32 + DESCRIPTION = "Unique point identifier for each jigsaw control point." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 2 + NAME = STATUS + DATA_TYPE = CHARACTER + UNIT = NONE + START_BYTE = 37 + BYTES = 12 + DESCRIPTION = "Status of point. It can be FREE, FIXED or CONSTRAINED." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 3 + NAME = ACCEPTED_MEASURES + DATA_TYPE = ASCII_INTEGER + UNIT = NONE + START_BYTE = 51 + BYTES = 4 + DESCRIPTION = "Number of accepted (non-rejected) measures for the control + point. This is actually the number of images that contain + the control point (also referred to as the point depth)." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 4 + NAME = REJECTED_MEASURES + DATA_TYPE = ASCII_INTEGER + UNIT = NONE + START_BYTE = 56 + BYTES = 4 + DESCRIPTION = "Number of measures rejected by jigsaw during the bundle + adjustment (if outlier rejection is active)" +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 5 + NAME = RESIDUAL_RMS + DATA_TYPE = ASCII_REAL + UNIT = PIXELS + START_BYTE = 61 + BYTES = 8 + DESCRIPTION = "Root mean square error of residuals for a point. + Indicates the statistical measure of variation in the + difference of each measure within a control point." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 6 + NAME = ADJUSTED_LATITUDE + DATA_TYPE = ASCII_REAL + UNIT = DEGREES + START_BYTE = 70 + BYTES = 16 + DESCRIPTION = "Latitude coordinate of the control point. This is the + adjusted location of the control point after jigsaw bundle + adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 7 + NAME = ADJUSTED_LONGITUDE + DATA_TYPE = ASCII_REAL + UNIT = DEGREES + START_BYTE = 87 + BYTES = 16 + DESCRIPTION = "Longitude coordinate of control point. This is the + adjusted location of the control point after jigsaw bundle + adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 8 + NAME = ADJUSTED_RADIUS + DATA_TYPE = ASCII_REAL + UNIT = KILOMETERS + START_BYTE = 104 + BYTES = 16 + DESCRIPTION = "Radius of control point. This is the adjusted radius of + the control point after jigsaw bundle adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 9 + NAME = SIGMA_LATITUDE + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 121 + BYTES = 16 + DESCRIPTION = "Adjusted uncertainty of Latitude coordinate of the control + point after jigsaw bundle adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 10 + NAME = SIGMA_LONGITUDE + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 138 + BYTES = 16 + DESCRIPTION = "Adjusted uncertainty of Longitude coordinate of the + control point after jigsaw bundle adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 11 + NAME = SIGMA_RADIUS + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 155 + BYTES = 16 + DESCRIPTION = "Adjusted uncertainty of Radius coordinate of the control + point after jigsaw bundle adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 12 + NAME = DELTA_LATITUDE + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 172 + BYTES = 16 + DESCRIPTION = "Latitude adjustment. The number of meters the latitude + coordinate has been adjusted by the jigsaw bundle + adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 13 + NAME = DELTA_LONGITUDE + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 189 + BYTES = 16 + DESCRIPTION = "Longitude adjustment. The number of meters the longitude + coordinate has been adjusted by the jigsaw bundle + adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 14 + NAME = DELTA_RADIUS + DATA_TYPE = ASCII_REAL + UNIT = METERS + START_BYTE = 206 + BYTES = 16 + DESCRIPTION = "Radius adjustment. The number of meters the radius has + been adjusted by the jigsaw bundle adjustment." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 15 + NAME = X + DATA_TYPE = ASCII_REAL + UNIT = KILOMETERS + START_BYTE = 223 + BYTES = 16 + DESCRIPTION = "Body-fixed X coordinate of the vector from the center of + the target body to the surface coordinate location." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 16 + NAME = Y + DATA_TYPE = ASCII_REAL + UNIT = KILOMETERS + START_BYTE = 240 + BYTES = 16 + DESCRIPTION = "Body-fixed Y coordinate of the vector from the center of + the target body to the surface coordinate location." +END_OBJECT = COLUMN + +OBJECT = COLUMN + COLUMN_NUMBER = 17 + NAME = Z + DATA_TYPE = ASCII_REAL + UNIT = KILOMETERS + START_BYTE = 257 + BYTES = 16 + DESCRIPTION = "Body-fixed Z coordinate of the vector from the center of + the target body to the surface coordinate location." +END_OBJECT = COLUMN +END diff --git a/messenger_wac_dem/README b/messenger_wac_dem/README new file mode 100644 index 0000000..cac0ff0 --- /dev/null +++ b/messenger_wac_dem/README @@ -0,0 +1,41 @@ +This directory contains the scripts used to create a WAC only DEM from all +MESSENGER orbital images. The first isisminer run is with the +mdis_image_overlaps.conf CONFIG file. It reads from an SQLite database +that is not generally available, but can (and was) created using output +from the caminfo application. The output of this is a 10x10 region based +set of CSV files of image overlap information. These files are then input +into the next isisminer run of mdis_wac_matcher.conf. After all these +files have been run, here is the list of the general steps to create a DEM: + + +1) Run all EW*_overlap.csv files in /scratch/kbecker/MESSENGER/Control/Overlaps + (i.e., find /scratch/kbecker/MESSENGER/Control/Overlaps -name 'EW*_overlaps.csv') + with mdis_wac_matcher.conf to create image-based control networks (from 118,267 + WAC image overlap sets). +2) Combine all these image-based files into a single global control network + using cnetcombinept. +3) Run cnetcheck of the global network to identify issues. +4) Run jigsaw to control images (will likely take a few days without error + propagation). +5) Run cnet2dem to create interpolated DEM. +6) Run series of noise and smoothing filters on DEM from previous step. +7) Run ckwriter on the updated images to generate CK kernel. + +Note, the control network is likely to be ginormous. You may need up to 512GB +of memory to complete these steps. Also, use of a compute cluster is highly +recommended! + +------------- + +To create a PDS table of the control point cloud, you can use the CONFIG +jigsaw_bundleout_sigmas_pdsfmt.conf. This requires the bundleout points file +created by jigsaw that *MUST* have error propagation turned on (without error +propagation, the format of the outfile is missing columns which will result +in very bad things!). The config file shows how POINTCLOUDTAB.FMT is created +but is disabled (by adding "_DNE" to the Strategy Object keyword) - this can +be time consuming. Here is the command used to create the MESSENGER PDS +pointcloud table: + +isisminer config=$ISIS3DATA/base/templates/isisminer/jigsaw_bundleout_sigmas_pdsfmt.conf \ + parameters="bundleout:RadAngTwist_global_jigsaw_errorprop_bundleout_points.csv \ + @tofmt:$ISIS3DATA/base/templates/isisminer/POINTCLOUDTAB.FMT@totab:MSGR_DEM_USG_EQ_C_V01.TAB" diff --git a/messenger_wac_dem/jigsaw_bundleout_sigmas_pdsfmt.conf b/messenger_wac_dem/jigsaw_bundleout_sigmas_pdsfmt.conf new file mode 100644 index 0000000..afd17f1 --- /dev/null +++ b/messenger_wac_dem/jigsaw_bundleout_sigmas_pdsfmt.conf @@ -0,0 +1,137 @@ +Object = IsisMiner + + Name = Jigsaw_Bundleout_Formatter + RequiredParameters = ( bundleout, tofmt, totab ) + + Object = Strategy + Name = CsvReader + Type = CsvReader + + # Here are the possible parameter + CsvFileArgs = BundleOut + CsvFile = "%1" + HasHeader = False + SkipLines = 3 + IgnoreComments = False + Delimiter = "," + Header = ("POINT_ID", "STATUS", + "ACCEPTED_MEASURES","REJECTED_MEASURES", "RESIDUAL_RMS", + "ADJUSTED_LATITUDE","ADJUSTED_LONGITUDE","ADJUSTED_RADIUS", + "SIGMA_LATITUDE","SIGMA_LONGITUDE","SIGMA_RADIUS", + "DELTA_LATITUDE", "DELTA_LONGITUDE", "DELTA_RADIUS", + "X", "Y", "Z") + Identity = Point_Id + + EndObject + + Object = Strategy_DNE + Name = JigsawBundleFormat + Type = PdsTableFormat + + # Here are the possible parameter + PdsFormatFileArgs = tofmt + PdsFormatFile = "%1" + Delimiter = "," + + # Specify the PDS table format COLUMNB objects + Column = ("POINT_ID", "STATUS", + "ACCEPTED_MEASURES","REJECTED_MEASURES", "RESIDUAL_RMS", + "ADJUSTED_LATITUDE","ADJUSTED_LONGITUDE","ADJUSTED_RADIUS", + "SIGMA_LATITUDE", "SIGMA_LONGITUDE", "SIGMA_RADIUS", + "DELTA_LATITUDE", "DELTA_LONGITUDE", "DELTA_RADIUS", + "X", "Y", "Z") + DataType = ("CHARACTER", "CHARACTER", + "ASCII_INTEGER", "ASCII_INTEGER", "ASCII_REAL", + "ASCII_REAL", "ASCII_REAL", "ASCII_REAL", + "ASCII_REAL", "ASCII_REAL", "ASCII_REAL", + "ASCII_REAL", "ASCII_REAL", "ASCII_REAL", + "ASCII_REAL", "ASCII_REAL", "ASCII_REAL") + Unit = ("NONE", "NONE", + "NONE", "NONE", "PIXELS", + "DEGREES", "DEGREES", "KILOMETERS", + "METERS", "METERS", "METERS", + "METERS", "METERS", "METERS", + "KILOMETERS", "KILOMETERS", "KILOMETERS") + DataDescription = ("Unique point identifier for each jigsaw control point.", + "Status of point. It can be FREE, FIXED or CONSTRAINED.", + "Number of accepted (non-rejected) measures for the control point. + This is actually the number of images that contain the control point + (also referred to as the point depth).", + "Number of measures rejected by jigsaw during the bundle + adjustment (if outlier rejection is active)", + "Root mean square error of residuals for a point. Indicates the + statistical measure of variation in the difference of each measure + within a control point.", + "Latitude coordinate of the control point. This is the + adjusted location of the control point after jigsaw bundle + adjustment.", + "Longitude coordinate of control point. This is the + adjusted location of the control point after jigsaw bundle + adjustment.", + "Radius of control point. This is the adjusted radius of + the control point after jigsaw bundle adjustment.", + "Adjusted uncertainty of Latitude coordinate of the control point + after jigsaw bundle adjustment.", + "Adjusted uncertainty of Longitude coordinate of the control point + after jigsaw bundle adjustment.", + "Adjusted uncertainty of Radius coordinate of the control point + after jigsaw bundle adjustment.", + "Latitude adjustment. The number of meters the latitude + coordinate has been adjusted by the jigsaw bundle + adjustment.", + "Longitude adjustment. The number of meters the longitude + coordinate has been adjusted by the jigsaw bundle + adjustment.", + "Radius adjustment. The number of meters the radius has been + adjusted by the jigsaw bundle adjustment.", + "Body-fixed X coordinate of the vector from the center of the + target body to the surface coordinate location.", + "Body-fixed Y coordinate of the vector from the center of the + target body to the surface coordinate location.", + "Body-fixed Z coordinate of the vector from the center of the + target body to the surface coordinate location.") + + + # Declare all column byte sizes + POINT_ID_BYTES = 32 + STATUS_BYTES = 12 + ACCEPTED_MEASURES_BYTES = 4 + REJECTED_MEASURES_BYTES = 4 + RESIDUAL_RMS_BYTES = 8 + ADJUSTED_LATITUDE_BYTES = 16 + ADJUSTED_LONGITUDE_BYTES = 16 + ADJUSTED_RADIUS_BYTES = 16 + SIGMA_LATITUDE_BYTES = 16 + SIGMA_LONGITUDE_BYTES = 16 + SIGMA_RADIUS_BYTES = 16 + DELTA_LATITUDE_BYTES = 16 + DELTA_LONGITUDE_BYTES = 16 + DELTA_RADIUS_BYTES = 16 + X_BYTES = 16 + Y_BYTES = 16 + Z_BYTES = 16 + + # Specify the column name that is unique to use as the Resource name + # (identity). If this is not present, an name is generated as "RowX" + # where X is input row starting at 0. + IdentityKey = POINT_ID + + EndObject + + Object=Strategy + Name = JigsawPointcloudWriter + Type = PdsTableCreator + + PdsFormatFileArgs = tofmt + PdsFormatFile = "%1" + + PdsTableFileArgs = totab + PdsTableFile = "%1" + + Mode = Create + Delimiter = "," + DefaultValue = "NULL" + EndObject +EndObject +End + diff --git a/messenger_wac_dem/mdis_image_overlaps.conf b/messenger_wac_dem/mdis_image_overlaps.conf new file mode 100644 index 0000000..2c37401 --- /dev/null +++ b/messenger_wac_dem/mdis_image_overlaps.conf @@ -0,0 +1,350 @@ +############################################################################### +# This isisminer CONFIG file determines image overlaps for all MDIS images. +# The complete set of MESSENGER MDIS image data is contained in two +# separate databases. This set of Strategies reads from two different +# databases created from footprintinit/caminfo data and determines the stereo +# compliments for every image in the defined Region. Region definitions +# should be used to restrict the range of images using database +# CenterLatitude/CenterLongitude and the corners of the Region definition in +# the GLOBALS file parameter. +# +# @author 2015-06-05 Kris Becker +# +############################################################################### +Object = IsisMiner + + Name = ImageOverlapper + RequiredParameters = ("DBDir", "OutputDir", "Region", "RegionLowerLeftLatitude", + "RegionUpperRightLatitude", "RegionLowerLeftLongitude", + "RegionUpperRightLongitude") + + Object = Strategy + # Read in orbital data before and including October 30, 2014 + Name = MDISSelectRegionOct2014 + Type = DatabaseReader + + DbFileArgs = "DBDir" + DbFile = "%1/Messenger_DOY_2011095_TO_2014304_gis.db" + Mode = Select + + # Select strings that need double quotes must use " since they cannot + # be stored in keyword values. These strings will be prescanned before + # issuing to the DB driver and they will be properly replaced. + Query = "select i.SourceProductId, i.YearDoy, i.File, i.Number, + i.Lines, i.Samples, i.TempDependentFocalLength, + g.StartTime, i.EtStartTime, i.ExposureDuration, + g.HasNorthPole, g.HasSouthPole, g.HasLongitudeBoundary, + g.CenterLongitude, g.CenterLatitude, g.CenterRadius, + i.ObservationType, g.PixelResolution, g.MeanGroundResolution, + g.IncidenceAngle, g.EmissionAngle, g.PhaseAngle, + g.SubSolarGroundAzimuth, g.SubSpacecraftGroundAzimuth, + g.SubSpacecraftLatitude, g.SubSpacecraftLongitude, + g.SubSolarLatitude, g.SubSolarLongitude, + g.TargetCenterDistance, g.SlantDistance, + g.ParallaxX, g.ParallaxY, g.ShadowX, g.ShadowY, + p.GisFootprint + from Polygon p + INNER JOIN Image i + ON p.SourceProductId=i.SourceProductId + INNER JOIN Geometry g + ON p.SourceProductId=g.SourceProductId" + + # Resource targets are for reading multiple rows from the DB. The fields + # from each row are created as Resources. + Target = Resource + Identity = "%1" + IdentityArgs = "SourceProductId" + + # Can specify a GIS keyword that will be converted on input + GisGeometryRef = GisFootprint + GisType = WKB + # Tolerance of 0.005 is about 213 at the Mercury equator + GisSimplifyTolerance = 0.005 + GisGeometryPointsKey = GisPoints + RemoveGisKeywordAfterImport = True + EndObject + + + Object = Strategy + # Read in orbital data after October 30, 2014 + Name = MDISSelectRegionMay2015 + Type = DatabaseReader + + DbFileArgs = "DBDir" + DbFile = "%1/Messenger_DOY_2014305_TO_2015120_gis.db" + Mode = Select + + # Select strings that need double quotes must use " since they cannot + # be stored in keyword values. These strings will be prescanned before + # issuing to the DB driver and they will be properly replaced. + Query = "select i.SourceProductId, i.YearDoy, i.File, i.Number, + i.Lines, i.Samples, i.TempDependentFocalLength, + g.StartTime, i.EtStartTime, i.ExposureDuration, + g.HasNorthPole, g.HasSouthPole, g.HasLongitudeBoundary, + g.CenterLongitude, g.CenterLatitude, g.CenterRadius, + i.ObservationType, g.PixelResolution, g.MeanGroundResolution, + g.IncidenceAngle, g.EmissionAngle, g.PhaseAngle, + g.SubSolarGroundAzimuth, g.SubSpacecraftGroundAzimuth, + g.SubSpacecraftLatitude, g.SubSpacecraftLongitude, + g.SubSolarLatitude, g.SubSolarLongitude, + g.TargetCenterDistance, g.SlantDistance, + g.ParallaxX, g.ParallaxY, g.ShadowX, g.ShadowY, + p.GisFootprint + from Polygon p + INNER JOIN Image i + ON p.SourceProductId=i.SourceProductId + INNER JOIN Geometry g + ON p.SourceProductId=g.SourceProductId" + + # Resource targets are for reading multiple rows from the DB. The fields + # from each row are created as Resources. + Target = Resource + Identity = "%1" + IdentityArgs = "SourceProductId" + + # Can specify a GIS keyword that will be converted on input + GisGeometryRef = GisFootprint + GisType = WKB + # Tolerance of 0.005 is about 213 at the Mercury equator + GisSimplifyTolerance = 0.005 + GisGeometryPointsKey = GisPoints + RemoveGisKeywordAfterImport = True + EndObject + + + Object = Strategy + # This algorithm discards images (Resources) where values in the Keyword + # value keys cannot be converted to double values. This will find the DB + # columns that have a NULL value. This specification will remove all + # images that don't have valid geometry at the image center (CenterLatitude, + # CenterLongitude). + Name = FindBadOnes + Type = IsNumeric + Description = "Identify images with invalid center pixel geometry" + Keyword = ( "CenterLatitude", "CenterLongitude" ) + EndObject + + + Object = Strategy + # The images identified in the previous Strategy cannot be use in any + # stereo computations so prevent their resurrection to help increase + # efficiency. + Name = DeleteBadOnes + Type = ResourceManager + Description = "Delete resources with bad values in required keywords" + Operations = DeleteDiscard + EndObject + + + Object = Strategy + # This algorithm will select only MDIS images where the center latitude + # and longitudes are within the defined region provided in the GLOBALS + # parameter file. + Name = SelectRegion + Type = Limit + Description = "Find all MDIS images with center image geometry in the + defined region" + + # This set of equations will result in 1 if test are good, meaning the + # clat/clon is in the defined region. If any test fails, InRegion = 0 + # (false) and the Resource is disabled. Note this also defines the GIS + # bounding box. + Group = Equations + InRegion = 1 + InRegion = "InRegion * (CenterLongitude > RegionLowerLeftLongitude)" + InRegion = "InRegion * (CenterLatitude > RegionLowerLeftLatitude)" + InRegion = "InRegion * (CenterLongitude <= RegionUpperRightLongitude)" + InRegion = "InRegion * (CenterLatitude <= RegionUpperRightLatitude)" + EndGroup + EndObject + + # Ensure the output directory exists for the CSV files + Object = Strategy + Name = MakeOutputDirectory + Type = RunCommand + Description = "Ensure output directories exists" + Debug = True + CommandArgs = ( "OutputDir", "Region" ) + Group = PreCommands + Mkdir1 = "mkdir -p %1/Images" + Mkdir2 = "mkdir -p %1/Overlaps/%2" + EndGroup + EndObject + + # Writes all files to CSV file that are in the region. It creates a file + # in the local directory with the name .csv where is the + # value contained in the Region definition file provided in GLOBALS + # paramater. + Object = Strategy + Name = WriteRegionImages + Type = CsvWriter + Description = "Write images that are in the defined region to a file" + CsvFile = "%1/Images/%2_images.csv" + CsvFileArgs = ( "OutputDir", "Region" ) + Mode = Create + Header = true + Keywords = (SourceProductId, YearDoy, File, Number, + Lines, Samples, TempDependentFocalLength, + StartTime, EtStartTime, ExposureDuration, + HasNorthPole, HasSouthPole, HasLongitudeBoundary, Region, + CenterLongitude, CenterLatitude, CenterRadius, ObservationType, + PixelResolution, MeanGroundResolution, + IncidenceAngle, EmissionAngle, PhaseAngle, + SubSolarGroundAzimuth, SubSpacecraftGroundAzimuth, + SubSpacecraftLongitude, SubSpacecraftLatitude, + SubSolarLongitude, SubSolarLatitude, + TargetCenterDistance, SlantDistance, + ParallaxX, ParallaxY, ShadowX, ShadowY, + GisPointsTolerance, GisPointsOriginal, GisPoints, GisFootprint) + GisGeometryKey = GisFootprint + GisType = WKB + Delimiter = "," + DefaultValue = "NULL" + EndObject + + Object = Strategy + # Determine images that overlaps for the selected ones + Name = ImageOverlap + Type = GisOverlap + Description = "Find all overlaps images in specfied region" + + Asset = ImageOverlaps + OverlapMerge = "ResourceB" +` + # Select candidates from input list. Matches over itself. This one set + # is used for all images that satisfy the regional limits. It can be + # dangerous to try to limit the candidates, however, using all images will + # make it slower since there are likely many more images to search + # through. + Object = CandidateMiner + Object = Strategy + Name = PairCandidates + Type = ResourceManager + Description = "Choose all resources to determine overlaping images" + Operations = ResetDiscard + EndObject + EndObject + +# Could specify a filename of the following form. +# OverlapConfigFile = "write_image_overlaps.conf" + +# This strategy is designed to process each Resource that has image overlaps. +# This approach will allow users to better manage the volume of data that +# will be generated from large runs of this operation. Specfically, the +# ImageOverlaps Asset can be removed from each Resource as soon as it is +# processed rather than wait till all are found and then results are written +# in the same manner. + Object = OverlapMiner + + Name = WriteImageOverlaps + RequiredParameters = ( "OutputDir", "Region" ) + + Object = Strategy + Name = WriteOverlapImages + Type = CsvWriter + CsvFile = "%1/Overlaps/%2/%3_overlaps.csv" + CsvFileArgs = ( "OutputDir", "Region", "SourceProductId" ) + Mode = Create + Header = true + Keywords = (SourceProductId, MatchedProductId, Region, + OverlapRatioSource, OverlapRatioMatch, + YearDoy, File, Number, Lines, Samples, TempDependentFocalLength, + StartTime, EtStartTime, ExposureDuration, + HasNorthPole, HasSouthPole, HasLongitudeBoundary, + CenterLongitude, CenterLatitude, CenterRadius, ObservationType, + PixelResolution, MeanGroundResolution, + IncidenceAngle, EmissionAngle, PhaseAngle, + SubSolarGroundAzimuth, SubSpacecraftGroundAzimuth, + SubSpacecraftLongitude, SubSpacecraftLatitude, + SubSolarLongitude, SubSolarLatitude, + TargetCenterDistance, SlantDistance, + ParallaxX, ParallaxY, ShadowX, ShadowY, + GisFootprint) + GisGeometryKey = GisFootprint + GisType = WKB + Delimiter = "," + DefaultValue = "NULL" + EndObject + + + # Write the image overlaps to a CSV file. The files will be put in a directory + # of the form "OutputDir/Region/SourceProductId_overlaps.csv" where + # OutputDir is provided in the program PARAMETERS as the full base path to + # the directory to write the files, and Region comes from the contents of + # the GLOBALS file. + # + # This shows the use of the AssetSidebar strategy. Note the "Asset" keyword + # is the same as the one specifed in StereoPair strategy above. All the + # Strategy Objects listed in the IsisMiner object (a whole other CONFIG file), + # will execute on any Resource that contains at least one stereo image pair. + # + # There is an Asset called ImageOverlaps, also a list of Resources, that just + # happens to work perfectly for all the following Strategies. Remember + # all Resource keywords containing the Asset are available in the global + # parameter list and can be referenced as any keyword. + + Object = Strategy + Name = WriteOverlapFiles + Type = AssetSidebar + Description = "Process all stereo pair assets" + Asset = ImageOverlaps + Operation = Append + + # Write all the overlapping images in the Resource + Object = IsisMiner + + Object = Strategy + # Sort the StereoPairs by rank + Name = SortByOverLapRatio + Type = NumericalSort + Description = "Sort images by overlap ratio" + SortKey = OverlapRatioB + Order = Descending + EndObject + + Object = Strategy + # Write result of StereoPair strategy for each Resource that has a + # StereoPairs asset. + Name = WriteOverlapSet + Type = CsvWriter + Description = "Write individual files with overlapping images" + Mode = Append + Header = False + SkipEmptyLists = True + + CsvFile = "%1/Overlaps/%2/%3_overlaps.csv" + CsvFileArgs = ( "OutputDir", "Region", "SourceProductId" ) + Keywords = (SourceProductIdB, SourceProductIdA, Region, + OverlapRatioB, OverlapRatioA, + YearDoyB, FileB, NumberB, LinesB, SamplesB, TempDependentFocalLengthB, + StartTimeB, EtStartTimeB, ExposureDurationB, + HasNorthPoleB, HasSouthPoleB, HasLongitudeBoundaryB, + CenterLongitudeB, CenterLatitudeB, CenterRadiusB, ObservationTypeB, + PixelResolutionB, MeanGroundResolutionB, + IncidenceAngleB, EmissionAngleB, PhaseAngleB, + SubSolarGroundAzimuthB, SubSpacecraftGroundAzimuthB, + SubSpacecraftLongitudeB, SubSpacecraftLatitudeB, + SubSolarLongitudeB, SubSolarLatitudeB, + TargetCenterDistanceB, SlantDistanceB, + ParallaxXB, ParallaxYB, ShadowXB, ShadowYB, + GisFootprint) + GisGeometryKey = GisFootprint + GisType = WKB + Delimiter = "," + DefaultValue = "NULL" + EndObject + EndObject + EndObject + + # Now remove the ImageOverlap asset from each Resource + Object = Strategy + Name = RemoveImageOverlaps + Type = ResourceManager + Description = "Delete ImageOverlaps asset to free memory (obsolete + merged Resources from overlaping pairs)" + Operations = DeleteAsset::ImageOverlaps + EndObject + EndObject + EndObject +EndObject +End diff --git a/messenger_wac_dem/mdis_wac_matcher.conf b/messenger_wac_dem/mdis_wac_matcher.conf new file mode 100644 index 0000000..0ccc413 --- /dev/null +++ b/messenger_wac_dem/mdis_wac_matcher.conf @@ -0,0 +1,428 @@ +#!/bin/csh + +# NOTE requires the C/T shell to run properly. However, one could create a +# global parameter file that defines the shell run command (i.e., +# EXECUTE="source" for C/T shell and EXECUTE="." for Bash) and provide this +# in isisminer's GLOBAL parameter. Then include EXECUTE in the CommandArgs +# keyword and its %n position in that keyword in the Matchem command in the +# MultiImageMatcher/SingleImageMatcher Strategies that runs the findfeatures +# commands. + +############################################################################### +# Prior steps to application of this script is to identify all images in +# a 10x10 degree region, for each image in each region determine all images +# that overlap it. There is another step that computes these input files +# using caminfo input. We used a database, the PVL files can be used as well. +# The result is a CSV file that contains many fields of data containing a list +# of every image that overlaps it. This file should be provided as the FROM +# parameter in isisnminer. +# +# This isisminer CONFIG file computes image-to-image control networks using +# feature-based matching of overlapping files. This config is designed to +# process all MDIS WAC images only. Limb images are identified by the +# observation type and excluded from processing. There are no other +# restrictions. +# +# High level processing is designed around the 10x10 degree regions that +# contain images whose center latitude and longitude coordinate fall within +# the region. Overlaps for all images are computed for the region images +# without restriction - i.e., all images that overlap individual region +# images are identified for matching. Currently, any image that has 3.5% +# of common overlap with one another is included for matching. +# +# This config takes a single overlap region file stored in CSV format. This +# file was created by an earlier run of isisminer that considered all orbital +# images as described above. It will filter out all NAC and limb image +# observations and execute the findfeatures application to simulatenously +# (using threads!) match all overlapping images using OpenCV feature matching +# algorithms. It creates a list of images that were not successfully +# matched in multi-match mode (must have >= 8 points) and runs them as pairs +# through findfeatures again with small adjustments to the paramters and has +# a Scharr filter applied to both images to enhance edges for better matching +# results. +# +# This processing results in a multi-match control and potentially many +# singleton (single match pair) networks. These are combined into one final +# network using the cnetcombinept app. All extraneous files can be deleted +# by uncommenting the proper commands. +# +# Once all regions have been processed, all image-based networks are combined +# into a single global network using cnetcombinept. Then jigsaw is ran to +# complete the control. +# +# INPUTS +# There are five required parameters to isisminer to execute this config. And +# certain requirements are assumed to be contained in of the overlaping CSV +# file. +# +# The description of the PARAMETERS required for this run are: +# +# multi_algorithms - contains the specification of the OpenCV feature +# matching algorithm applied in multi-match mode. +# +# single_algorithms - contains the specification of the OpenCV feature +# matching algorithm applied in singleton, pair-wise mode. +# +# imagedir - Specifies the root of the path to the MDIS image archive. +# Example: /scratch/kbecker/MESSENGER/Lev1 +# +# from - The input CSV file containing match file on the first row after the +# header and all subsequent rows contain images that overlap with +# it. +# Example: /scratch/kbecker/MESSENGER/Control/Overlaps/Equi025S305ED10/EW0210888537G_overlaps.csv +# +# outputdir - Path to write the image-based control networks. +# Example: /scratch/kbecker/MESSENGER/Control/WACOnly +# +# The full command to execute a single file in a region is (for berevity, +# let SCRIPTS=$ISIS3DATA/base/templates/isisminer and IN = /scratch/kbecker/MESSENGER): +# isisminer config=$SCRIPTS/mdis_wac_matcher.conf +# parameters="multi_algorithms:$SCRIPTS/multi_algorithms@ +# single_algorithms:$SCRIPTS/single_algorithms.lis@ +# imagedir:$IN/Lev1@ +# from:$IN/Lev1@ +# outputdir:$IN/Control/WACOnly" +# +# outputdir need not exist prior to processing but must be createable via +# the "mkdir -p" command. Directories that will be created in the script are: +# {outputdir}/{Region}_Networks - Region is of the form EquiLAT[S/N]LON[E/W]D10 +# +# In that directory will be the all the image-based networks for that 10x10 +# region of images. The file that will be combined into the image-based +# network are of the form Equi*_Networks/EW0210888537G_combined.net. +# +# @author 2016-02-18 Kris Becker +# +############################################################################### +Object = IsisMiner + + # This config is designed to work with overlap files generated using the + # GisOverlap strategy that creates a file of all overlapping images. + Name = MdisGlobalControl + RequiredParameters = ( "multi_algorithms", "single_algorithms", + "ImageDir", "From", "OutputDir" ) + + # Not applied but designed for these input files. The from paramter is a + # single overlap file. If a NAC overlap file is given in from, it will be + # ignored (see the SelectWACImages Filter Strategy). + Object = Defaults + Config = "$PWD/mdis_wac_matcher.conf" + Multi_Algorithms = "$PWD/multi_algorithms.lis" + Single_Algorithms = "$PWD/single_algorithms.lis" + ImageDir = "/scratch/kbecker/MESSENGER/Lev1" + From = "$PWD/Equi035N235ED10/*_overlaps.csv" + OutputDir = "$PWD" + EndObject + + # Reads overlap file where first line is match file, all other lines contain + # data for overlaping files. + Object = Strategy + Name = ReadImageOverlaps + Type = CsvReader + Description = "Read regional overlap image set" + + CsvFileArgs = "From" + CsvFile = "%1" + + # Reading the header sets the variable names for each column in each + # row (Resource) + HasHeader = True + Identity = "%1" + IdentityArgs = "SourceProductId" + EndObject + + # This strategy eliminates all NAC images from the list (which could be + # the match image). + Object = Strategy + # Select WAC images + Name = SelectWACImages + Type = Filter + Description = "Use RegExp to find all WAC only images" + Keyword = SourceProductId + RegExp = "^EW." + EndObject + + # Eliminate all limb observation images. + Object = Strategy + Name = NoLimbsPlease + Type = Filter + Description = "Eliminate all explicit limb observations..." + Keyword = ObservationType + Exclude = "Limb" + EndObject + + # Completely remove all images, including MATCH image, if they did not pass + # constraints in Limit::SelectWACImages and Filter::NoLimbsPlease. + Object = Strategy + Name = DeleteImagesNotPassingConstraints + Type = ResourceManager + Description = "Delete all images that do not pass the above constraints" + Operations = ( DeleteDiscard ) + EndObject + + # Find the FROMLIST image set that passed the constraints + Object = Strategy + Name = SelectMatchImage + Type = IsNumeric + Description = "(Non)Select MATCH image" + Keyword = OverlapRatioMatch + EndObject + + # See if we have a MATCH file. Won't do the AssetSidebar if it didn't pass + # constraints meaning the match image didn't make the cut. + Object = Strategy + Name = ChooseMatchImage + Type = ResourceManager + Description = "Select MATCH image" + Operations = ( ToggleDiscard ) + EndObject + + # This asset sidebar puts the MATCH file in the global variable pool and + # sets up environment to run the matcher with results set to FromList + Object = Strategy + Name = MultiMatcherLogic + Type = AssetSidebar + Asset = FromList + Operation = Create + CreateSource = Copy + + # Sidebar processing strategies. Toggle active/inactive get at match image + # and train images. This multi-strategy object applies multi-image + # matching. + Object = Isisminer + Name = MatcherLogic + + # Enable all remaining resources for overlap constraints testing + Object = Strategy + Name = ChooseMatchImage + Type = ResourceManager + Description = "Select MATCH image" + Operations = ( ResetDiscard ) + EndObject + + # Apply overlap ratio limits to all images (will implicity exclude MATCH + # image because of NULL in OverlapRatio* columns in the input CSV) + Object = Strategy + Name = ConstrainOverlapRatios + Type = Limit + Description = "Apply image constraints to all images" + + # This set of equations will result in 1 if test are good, meaning the + # constraints all pass. If any test fails, InRegion = 0 + # (false) and the Resource (overlapping image) is disabled. + Group = Equations + InRegion = 1 + # Needs at least a 3.50% overlap ratio (doesn't apply to MATCH as + # these values are not defined for it and it is implicitly discarded + # at this point) + InRegion = "InRegion * (OverlapRatioSource > 0.0350)" + InRegion = "InRegion * (OverlapRatioMatch > 0.0350)" + EndGroup + EndObject + + # Delete all non-passing resources which could also be the MATCH file + Object = Strategy + Name = RemoveBadOverlaps + Type = ResourceManager + Description = "Remove all bad overlaps for FROMLIST and the MATCH file" + Operations = ( DeleteDiscard ) + EndObject + + # Now just have FROMLIST Resources - create input conditions for matcher + Object = Strategy + Name = MultiImageMatcher + Type = RunCommand + Description = "Create findfeatures command and execute..." + + SkipCommandsIfNoData = True + OnPreCommandErrorContinue = False + OnPostCommandErrorContinue = False + + CommandArgs = ( ImageDir, OutputDir, Region, SourceProductId, + YearDoy, File, MatchedProductId, multi_algorithms ) + + # Setup commands for the findfeatures run. Got to be at least one FROM. + # Creates output directory {OutputDir}/{Region}_Networks. + Group = PreCommands + Mkdir = "mkdir -p %2/%3_Networks" + DeleteFrom = "/bin/rm -f %2/%3_Networks/%4_fromlist.lis" + DeleteCmd0 = "/bin/rm -f %2/%3_Networks/%4_matcher.cmd" + DeleteCmd1 = "/bin/rm -f %2/%3_Networks/%4_notmatched.lis" + Touch = "touch %2/%3_Networks/%4_fromlist.lis" + EndGroup + + # Append the current FROMLIST file to matcher input list + Group = Commands + Append = "echo "%1/%5/%6" >> %2/%3_Networks/%7_fromlist.lis" + EndGroup + + # Set up the multi-image matcher run and execute it + Group = PostCommands + Start = "echo "Making multi-matcher for %4..."" + Program = "echo "findfeatures + algospecfile=%8 + match=%1/%5/%6 fromlist=%2/%3_Networks/%4_fromlist.lis + fastgeom=true geomtype=camera + epitolerance=1.0 ratio=0.90 hmgtolerance=1.0 + networkid=%3 pointid='%4_?????' + onet=%2/%3_Networks/%4.net + tolist=%2/%3_Networks/%4_cubes.lis + tonotmatched=%2/%3_Networks/%4_notmatched.lis + description='Create image-image control network' + debug=false debuglog=%2/%3_Networks/%4.log" + > %2/%3_Networks/%4_matcher.cmd" + Report = "echo "Running multi-matcher for %4..."" + Matchem = "source %2/%3_Networks/%4_matcher.cmd" + Copyem = "cp -p %2/%3_Networks/%4.net %2/%3_Networks/%4_combined.net" + EndGroup + + EndObject + EndObject + + EndObject + + # Make some keywords for processing singleton networks + Object = Strategy + Name = KeywordMaker + Type = Calculator + Description = "Make some keys..." + + InitializersArgs = ( ImageDir, OutputDir, Region, SourceProductId, + YearDoy, File, MatchedProductId, multi_algorithms ) + Group = Initializers + SourceId = "%4" + Query = "%1/%5/%6" + EndGroup + + # Specify search ranges for cnetcombinept + Group = Equations + GroundTol = "PixelResolution * 3.0" + SearchRadius = "PixelResolution * 10.0" + EndGroup + + EndObject + + # With the MATCH file in the global variable pool here again, we will + # run through the list of failed ones individually and merge the networks + # when complete + Object = Strategy + Name = RunMatcher + Type = AssetSidebar + Asset = FailedList + Operation = Create + CreateSource = Copy + + Object = Isisminer + Name = SingleMatcherLogic + + # First clear all Resources and prepare to load remaining images to match + Object = Strategy + Name = ClearAll + Type = ResourceManager + Description = "Clear out all resources..." + Operations = ( ResetDiscard, ToggleDiscard, DeleteDiscard ) + EndObject + + # Read failed match file and process failed list with adjustments to + # algorithm parameters + Object = Strategy + Name = ReadFailedImages + Type = CsvReader + Description = "Read bad list for processing" + + CsvFileArgs = ( ImageDir, OutputDir, Region, SourceProductId, + YearDoy, File, MatchedProductId, single_algorithms, + SourceId ) + CsvFile = "%2/%3_Networks/%4_notmatched.lis" + + # Its just a list of image file names + HasHeader = false + ColumnBaseName = "Column" + RowBaseName = "S" + EndObject + + # Now just have FROMLIST Resources - create input conditions for matcher. + # This set of commands runs the findfeature algorithms for pair-wise + # singleton networks. + # + # This strategy will take the truth image and run single pair matching + # with additional parameters. A Scharr filter is run on both images to + # enhance edge detection. Successful pair-wise matching results in output + # files of the form {OutputDir}/{Region}_Networks/{SourceProductId}_S#.net + # where # corresponds to the row number of images listed in the notmatch + # file in the previous multi-match mode. + Object = Strategy + Name = SingleImageMatcher + Type = RunCommand + Description = "Run on each failed image with special parameters" + + SkipCommandsIfNoData = True + OnPreCommandErrorContinue = False + OnPostCommandErrorContinue = False + + # Parameters substitutions for all commands + CommandArgs = ( ImageDir, OutputDir, Region, SourceProductId, + YearDoy, File, MatchedProductId, single_algorithms, + Column0, Identity, SourceId , Query, SearchRadius, + GroundTol ) + + # Setup commands for the findfeatures run. Got to be at least one FROM. + Group = PreCommands + DeleteCmd0 = "/bin/rm -f %2/%3_Networks/%4_%10_matcher.cmd" + DeleteCmd1 = "/bin/rm -f %2/%3_Networks/%4_singles_notmatched.lis" + Touch = "touch %2/%3_Networks/%4_singlenets.lis" + EndGroup + + # Match single pairs (singletons) of images directly + Group = Commands + Report = "echo "Making single matcher on %9..."" + Program = "echo "findfeatures + algospecfile=%8 + match=%12 from=%9 + fastgeom=true geomtype=crop filter=scharr + epitolerance=1.0 ratio=0.99 hmgtolerance=1.0 + networkid=%3_%10 pointid='%4_%10_?????' + onet=%2/%3_Networks/%4_%10.net + tonotmatched=%2/%3_Networks/%4_singles_notmatched.lis + description='Create image-image control network' + debug=false debuglog=%2/%3_Networks/%4_%10.log" + > %2/%3_Networks/%4_%10_matcher.cmd" + # Execute the single matcher command + Report = "echo "Running single matcher for %4..."" + Matchem = "source %2/%3_Networks/%4_%10_matcher.cmd" + # Append to network cubes list + AddCube = "echo "%9" >> %2/%3_Networks/%4_cubes.lis" + AddNet = "echo "%2/%3_Networks/%4_%10.net" + >> %2/%3_Networks/%4_singlenets.lis" +# Files may not be needed/used +# Cleanup = "/bin/rm -f %2/%3_Networks/%4_%10_matcher.cmd +# %2/%3_Networks/%4_%10.log" + EndGroup + + # Combine singleton networks into one net with 3 measure mininum. Then + # combine singletons with initial multi-match network w/2 measure + # minimum + Group = PostCommands + Combine1 = "cnetcombinept + cnetlist=%2/%3_Networks/%4_singlenets.lis + imagetol=1.0 + distance=%13 + minmeasures=3 + onet=%2/%3_Networks/%4_singletons.net" + # Combine comprehensive singleton network to multi-matched network + Combine2 = "cnetcombinept + cnetbase=%2/%3_Networks/%4.net + cnetfrom=%2/%3_Networks/%4_singletons.net + imagetol=1.0 + distance=%13 + groundtol=%14 + onet=%2/%3_Networks/%4_combined.net " + EndGroup + + EndObject + EndObject + EndObject +EndObject + +End + diff --git a/messenger_wac_dem/multi_algorithms.lis b/messenger_wac_dem/multi_algorithms.lis new file mode 100644 index 0000000..f101442 --- /dev/null +++ b/messenger_wac_dem/multi_algorithms.lis @@ -0,0 +1,4 @@ +# Generalized matcher algorithm. Uses a FASTX edge detector and SIFT extractor +# with the default matcher (BFMatcher(L2_NORM)). The maxpoints parameter +# restricts the number of initial detected features before outlier detection. +detector.fastx@threshold:25@type:2/sift/parameters@maxpoints:400 diff --git a/messenger_wac_dem/single_algorithms.lis b/messenger_wac_dem/single_algorithms.lis new file mode 100644 index 0000000..4fc395b --- /dev/null +++ b/messenger_wac_dem/single_algorithms.lis @@ -0,0 +1,4 @@ +# Single matcher algorithm. Designed to be used with a Scharr filter. Using +# the crop function will further limit the number of features detected. +detector.fastx@threshold:17@type:2/sift/parameters@geomtype:crop@filter:scharr@maxpoints:2000 +