From 0652813d3f1eda4af3b8ae5054b19b88d068d488 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Mon, 6 Nov 2023 10:23:27 -0700 Subject: [PATCH 01/18] Add hexit binary project This binary is for converting nasadem tiles into H3 polyfills --- Cargo.toml | 1 + hexit/Cargo.toml | 18 ++++++++ hexit/src/combine.rs | 10 ++++ hexit/src/main.rs | 15 ++++++ hexit/src/options.rs | 49 ++++++++++++++++++++ hexit/src/tesselate.rs | 102 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+) create mode 100644 hexit/Cargo.toml create mode 100644 hexit/src/combine.rs create mode 100644 hexit/src/main.rs create mode 100644 hexit/src/options.rs create mode 100644 hexit/src/tesselate.rs diff --git a/Cargo.toml b/Cargo.toml index 7e0710b..804ce5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "geopath", + "hexit", "itm", "nasadem", "propah", diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml new file mode 100644 index 0000000..12d3b32 --- /dev/null +++ b/hexit/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "hexit" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +byteorder = { workspace = true } +clap = { version = "4.4.2", features = ["derive"] } +env_logger = "0.10" +flate2 = "1.0.28" +geo = { workspace = true } +h3o = { version = "0.4.0", features = ["geo"] } +indicatif = "0.17.7" +itertools = "0.10" +nasadem = { path = "../nasadem" } +num-traits = { workspace = true } +rayon = "1.8.0" diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs new file mode 100644 index 0000000..a1ca07f --- /dev/null +++ b/hexit/src/combine.rs @@ -0,0 +1,10 @@ +use crate::options::Combine; +use anyhow::Result; + +impl Combine { + pub fn run(&self) -> Result<()> { + assert!(!self.input.is_empty()); + let res: Result<()> = Ok(()); + res + } +} diff --git a/hexit/src/main.rs b/hexit/src/main.rs new file mode 100644 index 0000000..a04bc06 --- /dev/null +++ b/hexit/src/main.rs @@ -0,0 +1,15 @@ +mod combine; +mod options; +mod tesselate; + +use anyhow::Result; +use clap::Parser; +use options::Cli; + +fn main() -> Result<()> { + let cli = Cli::parse(); + match cli { + Cli::Tessellate(tesselate) => tesselate.run(), + Cli::Combine(combine) => combine.run(), + } +} diff --git a/hexit/src/options.rs b/hexit/src/options.rs new file mode 100644 index 0000000..f073891 --- /dev/null +++ b/hexit/src/options.rs @@ -0,0 +1,49 @@ +use clap::{Args, Parser}; +use h3o::Resolution; +use std::path::PathBuf; + +/// Generate H3 tessellated polyfills from raster data. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub enum Cli { + /// Generate a tessellated list of (cell, elevation) for each + /// input file. + Tessellate(Tesselate), + + /// Combine previously tesselated files into a single + Combine(Combine), +} + +#[derive(Debug, Clone, Args)] +pub struct Tesselate { + /// Reprocess height file even if corresponding output already + /// exists. + #[arg(short = 'O', long)] + pub overwrite: bool, + + /// Amount of compression. + #[arg(short, long, default_value_t = 6)] + pub compression: u32, + + #[arg(short, long, default_value_t = Resolution::Twelve)] + pub resolution: Resolution, + + /// Output directory. + #[arg(short, long)] + pub out_dir: PathBuf, + + /// Input SRTM elevation (.hgt) files. + pub input: Vec, +} + +#[derive(Debug, Clone, Args)] +pub struct Combine { + #[arg(short, long)] + pub resolution: Resolution, + + #[arg(short, long)] + pub out_path: PathBuf, + + /// Input tessaltions. + pub input: Vec, +} diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs new file mode 100644 index 0000000..9161f20 --- /dev/null +++ b/hexit/src/tesselate.rs @@ -0,0 +1,102 @@ +use crate::options::Tesselate; +use anyhow::Result; +use byteorder::{LittleEndian as LE, WriteBytesExt}; +use flate2::{write::GzEncoder, Compression}; +use h3o::{ + geom::{PolyfillConfig, Polygon, ToCells}, + CellIndex, Resolution, +}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use nasadem::{Sample, Tile}; +use rayon::prelude::*; +use std::{ + fs::{self, File}, + io::{BufWriter, Write}, + path::Path, +}; + +impl Tesselate { + pub fn run(&self) -> Result<()> { + let progress_group = MultiProgress::new(); + self.input + .par_iter() + .try_for_each(|height_file_path| self._run(height_file_path, &progress_group))?; + Ok(()) + } + + fn _run(&self, height_file_path: &Path, progress_group: &MultiProgress) -> Result<()> { + let out_file_name = { + let file_name = height_file_path + .file_name() + .expect("we already parsed the tile, therefore path must be a file") + .to_str() + .expect("we already parsed the tile, therefore path must be a file"); + format!("{file_name}.res{}.h3tez", self.resolution) + }; + let out_file_path = self.out_dir.clone().join(&out_file_name); + + if out_file_path.exists() && !self.overwrite { + // Exit early if we've already processed this input. + return Ok(()); + } + + let out_file_tmp_path = { + let mut p = out_file_path.clone(); + p.set_extension("tmp"); + p + }; + + let tile = Tile::memmap(height_file_path)?; + let progress_bar = make_progress_bar(out_file_name, tile.len() as u64); + let tmp_out_file = File::create(&out_file_tmp_path)?; + let tmp_out_wtr = GzEncoder::new(tmp_out_file, Compression::new(self.compression)); + self.polyfill_tile( + &tile, + &progress_group.add(progress_bar), + BufWriter::new(tmp_out_wtr), + )?; + fs::rename(out_file_tmp_path, out_file_path)?; + Ok(()) + } + + fn polyfill_tile( + &self, + tile: &Tile, + progress_bar: &ProgressBar, + mut out: impl Write, + ) -> Result<()> { + for sample in tile.iter() { + let (elev, hexes) = polyfill_sample(&sample, self.resolution)?; + out.write_i16::(elev)?; + out.write_u16::(u16::try_from(hexes.len())?)?; + for hex in hexes { + out.write_u64::(hex)?; + } + progress_bar.inc(1); + } + Ok(()) + } +} + +fn polyfill_sample(sample: &Sample, resolution: Resolution) -> Result<(i16, Vec)> { + let elevation = sample.elevation(); + let polygon = Polygon::from_degrees(sample.polygon())?; + let cell_iter = polygon.to_cells(PolyfillConfig::new(resolution)); + let mut cells: Vec = CellIndex::compact(cell_iter)?.map(u64::from).collect(); + cells.sort_unstable(); + cells.dedup(); + Ok((elevation, cells)) +} + +/// Returns a progress bar object for the given parquet file and name. +fn make_progress_bar(prefix: String, total_size: u64) -> ProgressBar { + #[allow(clippy::cast_sign_loss)] + let pb = ProgressBar::new(total_size); + pb.set_prefix(prefix); + pb.set_style( + ProgressStyle::with_template("{prefix}...\n[{wide_bar:.cyan/blue}]") + .expect("incorrect progress bar format string") + .progress_chars("#>-"), + ); + pb +} From f7e1ee127b09f3849c5a3108864c917c7d3b7b29 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 10:17:11 -0700 Subject: [PATCH 02/18] Add combine command --- hexit/Cargo.toml | 1 + hexit/src/combine.rs | 97 +++++++++++++++++++++++++++++++++++++-- hexit/src/main.rs | 1 + hexit/src/options.rs | 4 +- hexit/src/progress_bar.rs | 14 ++++++ hexit/src/tesselate.rs | 18 ++------ 6 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 hexit/src/progress_bar.rs diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml index 12d3b32..d82ab64 100644 --- a/hexit/Cargo.toml +++ b/hexit/Cargo.toml @@ -11,6 +11,7 @@ env_logger = "0.10" flate2 = "1.0.28" geo = { workspace = true } h3o = { version = "0.4.0", features = ["geo"] } +hextree = { git = "https://github.com/JayKickliter/HexTree.git", branch = "jsk/add-disk-repr", features = ["disktree"] } indicatif = "0.17.7" itertools = "0.10" nasadem = { path = "../nasadem" } diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index a1ca07f..14cabbf 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,10 +1,101 @@ -use crate::options::Combine; +use crate::{options::Combine, progress_bar::make_progress_bar}; use anyhow::Result; +use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; +use flate2::bufread::GzDecoder; +use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap}; +use indicatif::MultiProgress; +use std::{fs::File, io::BufReader, path::Path}; impl Combine { pub fn run(&self) -> Result<()> { assert!(!self.input.is_empty()); - let res: Result<()> = Ok(()); - res + let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); + let progress_group = MultiProgress::new(); + for tess_file_path in &self.input { + Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; + } + self.write_disktree(&hextree, &progress_group)?; + self.verify_disktree(&hextree, &progress_group)?; + Ok(()) + } + + fn read_tessellation( + tess_file_path: &Path, + progress_group: &MultiProgress, + hextree: &mut HexTreeMap, + ) -> Result<()> { + let tess_file = File::open(tess_file_path)?; + let tess_buf_rdr = BufReader::new(tess_file); + let mut rdr = GzDecoder::new(tess_buf_rdr); + let tess_file_name = tess_file_path + .file_name() + .expect("we already parsed the tile, therefore path must be a file") + .to_str() + .expect("we already parsed the tile, therefore path must be a file"); + + let n_samples = rdr.read_u64::()?; + let pb = progress_group.add(make_progress_bar(tess_file_name.to_string(), n_samples)); + for _sample_n in 0..n_samples { + let elevation = rdr.read_i16::()?; + let n_cells = rdr.read_u16::()?; + for _cell_n in 0..n_cells { + let raw_cell = rdr.read_u64::()?; + let cell = hextree::Cell::from_raw(raw_cell)?; + hextree.insert(cell, elevation); + } + pb.inc(1); + } + assert!( + rdr.read_u8().is_err(), + "We should have read all samples out of the file" + ); + + Ok(()) + } + + fn write_disktree( + &self, + hextree: &HexTreeMap, + progress_group: &MultiProgress, + ) -> Result<()> { + let disktree_file = File::create(&self.out)?; + let disktree_file_name = self + .out + .file_name() + .expect("we already parsed the tile, therefore path must be a file") + .to_str() + .expect("we already parsed the tile, therefore path must be a file"); + let disktree_len = hextree.len(); + let pb = make_progress_bar(disktree_file_name.to_string(), disktree_len as u64); + let pb = progress_group.add(pb); + hextree.to_disktree(disktree_file, |wtr, val| { + pb.inc(1); + wtr.write_i16::(*val) + })?; + Ok(()) + } + + fn verify_disktree( + &self, + hextree: &HexTreeMap, + progress_group: &MultiProgress, + ) -> Result<()> { + fn value_reader(res: hextree::Result<(Cell, &mut File)>) -> Result<(Cell, i16)> { + let (cell, rdr) = res?; + Ok(rdr.read_i16::().map(|val| (cell, val))?) + } + + let mut disktree = DiskTree::open(&self.out)?; + let pb = make_progress_bar("Validating disktree".to_string(), hextree.len() as u64); + let pb = progress_group.add(pb); + let mut count = 0; + for res in disktree.iter(value_reader)? { + let (cell, value) = res?; + assert_eq!(Some((cell, &value)), hextree.get(cell)); + pb.inc(1); + count += 1; + } + assert_eq!(hextree.len(), count); + Ok(()) } } diff --git a/hexit/src/main.rs b/hexit/src/main.rs index a04bc06..34ee480 100644 --- a/hexit/src/main.rs +++ b/hexit/src/main.rs @@ -1,5 +1,6 @@ mod combine; mod options; +mod progress_bar; mod tesselate; use anyhow::Result; diff --git a/hexit/src/options.rs b/hexit/src/options.rs index f073891..09ff3f6 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -38,11 +38,11 @@ pub struct Tesselate { #[derive(Debug, Clone, Args)] pub struct Combine { - #[arg(short, long)] + #[arg(short, long, default_value_t = Resolution::Ten)] pub resolution: Resolution, #[arg(short, long)] - pub out_path: PathBuf, + pub out: PathBuf, /// Input tessaltions. pub input: Vec, diff --git a/hexit/src/progress_bar.rs b/hexit/src/progress_bar.rs new file mode 100644 index 0000000..fb30c4e --- /dev/null +++ b/hexit/src/progress_bar.rs @@ -0,0 +1,14 @@ +use indicatif::{ProgressBar, ProgressStyle}; + +/// Returns a progress bar object for the given parquet file and name. +pub fn make_progress_bar(prefix: String, total_size: u64) -> ProgressBar { + #[allow(clippy::cast_sign_loss)] + let pb = ProgressBar::new(total_size); + pb.set_prefix(prefix); + pb.set_style( + ProgressStyle::with_template("{prefix}...\n[{wide_bar:.cyan/blue}]") + .expect("incorrect progress bar format string") + .progress_chars("#>-"), + ); + pb +} diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 9161f20..422f34a 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -1,4 +1,4 @@ -use crate::options::Tesselate; +use crate::{options::Tesselate, progress_bar::make_progress_bar}; use anyhow::Result; use byteorder::{LittleEndian as LE, WriteBytesExt}; use flate2::{write::GzEncoder, Compression}; @@ -6,7 +6,7 @@ use h3o::{ geom::{PolyfillConfig, Polygon, ToCells}, CellIndex, Resolution, }; -use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use indicatif::{MultiProgress, ProgressBar}; use nasadem::{Sample, Tile}; use rayon::prelude::*; use std::{ @@ -65,6 +65,7 @@ impl Tesselate { progress_bar: &ProgressBar, mut out: impl Write, ) -> Result<()> { + out.write_u64::(tile.len() as u64)?; for sample in tile.iter() { let (elev, hexes) = polyfill_sample(&sample, self.resolution)?; out.write_i16::(elev)?; @@ -87,16 +88,3 @@ fn polyfill_sample(sample: &Sample, resolution: Resolution) -> Result<(i16, Vec< cells.dedup(); Ok((elevation, cells)) } - -/// Returns a progress bar object for the given parquet file and name. -fn make_progress_bar(prefix: String, total_size: u64) -> ProgressBar { - #[allow(clippy::cast_sign_loss)] - let pb = ProgressBar::new(total_size); - pb.set_prefix(prefix); - pb.set_style( - ProgressStyle::with_template("{prefix}...\n[{wide_bar:.cyan/blue}]") - .expect("incorrect progress bar format string") - .progress_chars("#>-"), - ); - pb -} From 9a78a80e2b0ce1bf49229d1cc4bb25af750efecf Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 10:45:19 -0700 Subject: [PATCH 03/18] Add lookuo command --- hexit/src/combine.rs | 16 +++++++++------ hexit/src/lookup.rs | 23 ++++++++++++++++++++++ hexit/src/main.rs | 4 +++- hexit/src/options.rs | 15 +++++++++++--- hexit/src/{progress_bar.rs => progress.rs} | 9 ++++----- hexit/src/tesselate.rs | 10 +++------- 6 files changed, 55 insertions(+), 22 deletions(-) create mode 100644 hexit/src/lookup.rs rename hexit/src/{progress_bar.rs => progress.rs} (50%) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 14cabbf..230988e 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,4 +1,4 @@ -use crate::{options::Combine, progress_bar::make_progress_bar}; +use crate::{options::Combine, progress}; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use flate2::bufread::GzDecoder; @@ -34,7 +34,7 @@ impl Combine { .expect("we already parsed the tile, therefore path must be a file"); let n_samples = rdr.read_u64::()?; - let pb = progress_group.add(make_progress_bar(tess_file_name.to_string(), n_samples)); + let pb = progress_group.add(progress::bar(tess_file_name.to_string(), n_samples)); for _sample_n in 0..n_samples { let elevation = rdr.read_i16::()?; let n_cells = rdr.read_u16::()?; @@ -66,8 +66,10 @@ impl Combine { .to_str() .expect("we already parsed the tile, therefore path must be a file"); let disktree_len = hextree.len(); - let pb = make_progress_bar(disktree_file_name.to_string(), disktree_len as u64); - let pb = progress_group.add(pb); + let pb = progress_group.add(progress::bar( + disktree_file_name.to_string(), + disktree_len as u64, + )); hextree.to_disktree(disktree_file, |wtr, val| { pb.inc(1); wtr.write_i16::(*val) @@ -86,8 +88,10 @@ impl Combine { } let mut disktree = DiskTree::open(&self.out)?; - let pb = make_progress_bar("Validating disktree".to_string(), hextree.len() as u64); - let pb = progress_group.add(pb); + let pb = progress_group.add(progress::bar( + "Validating disktree".to_string(), + hextree.len() as u64, + )); let mut count = 0; for res in disktree.iter(value_reader)? { let (cell, value) = res?; diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs new file mode 100644 index 0000000..7586b3a --- /dev/null +++ b/hexit/src/lookup.rs @@ -0,0 +1,23 @@ +use crate::options::Lookup; +use anyhow::Result; +use byteorder::{LittleEndian as LE, ReadBytesExt}; +use hextree::{disktree::DiskTree, Cell}; + +impl Lookup { + pub fn run(&self) -> Result<()> { + let raw_cell: u64 = self + .cell + .parse::() + .or_else(|_| u64::from_str_radix(&self.cell, 16))?; + let cell = Cell::try_from(raw_cell)?; + let mut disktree = DiskTree::open(&self.disktree)?; + match disktree.seek_to_cell(cell)? { + None => (), + Some((cell, rdr)) => { + let val = rdr.read_i16::()?; + println!("{cell}: {val}"); + } + } + Ok(()) + } +} diff --git a/hexit/src/main.rs b/hexit/src/main.rs index 34ee480..89098bb 100644 --- a/hexit/src/main.rs +++ b/hexit/src/main.rs @@ -1,6 +1,7 @@ mod combine; +mod lookup; mod options; -mod progress_bar; +mod progress; mod tesselate; use anyhow::Result; @@ -12,5 +13,6 @@ fn main() -> Result<()> { match cli { Cli::Tessellate(tesselate) => tesselate.run(), Cli::Combine(combine) => combine.run(), + Cli::Lookup(lookup) => lookup.run(), } } diff --git a/hexit/src/options.rs b/hexit/src/options.rs index 09ff3f6..c74d766 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -6,14 +6,15 @@ use std::path::PathBuf; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] pub enum Cli { - /// Generate a tessellated list of (cell, elevation) for each - /// input file. Tessellate(Tesselate), - /// Combine previously tesselated files into a single Combine(Combine), + + Lookup(Lookup), } +/// Generate a tessellated list of (cell, elevation) for each +/// input file. #[derive(Debug, Clone, Args)] pub struct Tesselate { /// Reprocess height file even if corresponding output already @@ -36,6 +37,7 @@ pub struct Tesselate { pub input: Vec, } +/// Combine previously tesselated files into a single #[derive(Debug, Clone, Args)] pub struct Combine { #[arg(short, long, default_value_t = Resolution::Ten)] @@ -47,3 +49,10 @@ pub struct Combine { /// Input tessaltions. pub input: Vec, } + +/// Lookup value for H3 cell in a disktree. +#[derive(Debug, Clone, Args)] +pub struct Lookup { + pub disktree: PathBuf, + pub cell: String, +} diff --git a/hexit/src/progress_bar.rs b/hexit/src/progress.rs similarity index 50% rename from hexit/src/progress_bar.rs rename to hexit/src/progress.rs index fb30c4e..1fddba6 100644 --- a/hexit/src/progress_bar.rs +++ b/hexit/src/progress.rs @@ -1,10 +1,9 @@ use indicatif::{ProgressBar, ProgressStyle}; -/// Returns a progress bar object for the given parquet file and name. -pub fn make_progress_bar(prefix: String, total_size: u64) -> ProgressBar { - #[allow(clippy::cast_sign_loss)] - let pb = ProgressBar::new(total_size); - pb.set_prefix(prefix); +pub fn bar(header: String, length: u64) -> ProgressBar { + let pb = ProgressBar::hidden(); + pb.set_prefix(header); + pb.set_length(length); pb.set_style( ProgressStyle::with_template("{prefix}...\n[{wide_bar:.cyan/blue}]") .expect("incorrect progress bar format string") diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 422f34a..e7e99c1 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -1,4 +1,4 @@ -use crate::{options::Tesselate, progress_bar::make_progress_bar}; +use crate::{options::Tesselate, progress}; use anyhow::Result; use byteorder::{LittleEndian as LE, WriteBytesExt}; use flate2::{write::GzEncoder, Compression}; @@ -47,14 +47,10 @@ impl Tesselate { }; let tile = Tile::memmap(height_file_path)?; - let progress_bar = make_progress_bar(out_file_name, tile.len() as u64); + let pb = progress_group.add(progress::bar(out_file_name, tile.len() as u64)); let tmp_out_file = File::create(&out_file_tmp_path)?; let tmp_out_wtr = GzEncoder::new(tmp_out_file, Compression::new(self.compression)); - self.polyfill_tile( - &tile, - &progress_group.add(progress_bar), - BufWriter::new(tmp_out_wtr), - )?; + self.polyfill_tile(&tile, &pb, BufWriter::new(tmp_out_wtr))?; fs::rename(out_file_tmp_path, out_file_path)?; Ok(()) } From d76ef02f0505932b45b01ee4901962a6dab5aeb3 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 16:00:50 -0700 Subject: [PATCH 04/18] Log seek time --- hexit/src/lookup.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index 7586b3a..91f6a73 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -11,11 +11,15 @@ impl Lookup { .or_else(|_| u64::from_str_radix(&self.cell, 16))?; let cell = Cell::try_from(raw_cell)?; let mut disktree = DiskTree::open(&self.disktree)?; + let t0 = std::time::Instant::now(); match disktree.seek_to_cell(cell)? { None => (), Some((cell, rdr)) => { + let t_seek = t0.elapsed(); let val = rdr.read_i16::()?; + let t_tot = t0.elapsed(); println!("{cell}: {val}"); + println!("{t_seek:?} {t_tot:?}"); } } Ok(()) From 8fcb8cc9b2fdf76c37b91add0f3006470ba09d53 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 17:44:11 -0700 Subject: [PATCH 05/18] Add geomasking for filtering only desired areas --- hexit/Cargo.toml | 1 + hexit/src/combine.rs | 28 +++++++++++++++++++--------- hexit/src/main.rs | 1 + hexit/src/mask.rs | 16 ++++++++++++++++ hexit/src/options.rs | 12 ++++++++++++ hexit/src/tesselate.rs | 38 +++++++++++++++++++++++++------------- 6 files changed, 74 insertions(+), 22 deletions(-) create mode 100644 hexit/src/mask.rs diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml index d82ab64..bfbdad6 100644 --- a/hexit/Cargo.toml +++ b/hexit/Cargo.toml @@ -10,6 +10,7 @@ clap = { version = "4.4.2", features = ["derive"] } env_logger = "0.10" flate2 = "1.0.28" geo = { workspace = true } +geojson = "0.24.1" h3o = { version = "0.4.0", features = ["geo"] } hextree = { git = "https://github.com/JayKickliter/HexTree.git", branch = "jsk/add-disk-repr", features = ["disktree"] } indicatif = "0.17.7" diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 230988e..9694a7b 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,7 +1,9 @@ -use crate::{options::Combine, progress}; +use crate::{mask, options::Combine, progress}; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use flate2::bufread::GzDecoder; +use geo::{coord, GeometryCollection, Intersects}; +use h3o::{CellIndex, LatLng}; use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap}; use indicatif::MultiProgress; use std::{fs::File, io::BufReader, path::Path}; @@ -11,8 +13,9 @@ impl Combine { assert!(!self.input.is_empty()); let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); let progress_group = MultiProgress::new(); + let mask = mask::open(self.mask.as_deref())?; for tess_file_path in &self.input { - Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; + Self::read_tessellation(tess_file_path, &progress_group, mask.as_ref(), &mut hextree)?; } self.write_disktree(&hextree, &progress_group)?; self.verify_disktree(&hextree, &progress_group)?; @@ -22,6 +25,7 @@ impl Combine { fn read_tessellation( tess_file_path: &Path, progress_group: &MultiProgress, + mask: Option<&GeometryCollection>, hextree: &mut HexTreeMap, ) -> Result<()> { let tess_file = File::open(tess_file_path)?; @@ -29,9 +33,8 @@ impl Combine { let mut rdr = GzDecoder::new(tess_buf_rdr); let tess_file_name = tess_file_path .file_name() - .expect("we already parsed the tile, therefore path must be a file") - .to_str() - .expect("we already parsed the tile, therefore path must be a file"); + .and_then(|n| n.to_str()) + .expect("already opened, therefore path must be a file"); let n_samples = rdr.read_u64::()?; let pb = progress_group.add(progress::bar(tess_file_name.to_string(), n_samples)); @@ -41,7 +44,15 @@ impl Combine { for _cell_n in 0..n_cells { let raw_cell = rdr.read_u64::()?; let cell = hextree::Cell::from_raw(raw_cell)?; - hextree.insert(cell, elevation); + let mask_contains_cell = mask.as_ref().map_or(true, |mask| { + let cell_index = CellIndex::try_from(cell.into_raw()).unwrap(); + let cell_center = LatLng::from(cell_index); + let coord = coord!(x: cell_center.lng(), y: cell_center.lat()); + mask.intersects(&coord) + }); + if mask_contains_cell { + hextree.insert(cell, elevation); + } } pb.inc(1); } @@ -62,9 +73,8 @@ impl Combine { let disktree_file_name = self .out .file_name() - .expect("we already parsed the tile, therefore path must be a file") - .to_str() - .expect("we already parsed the tile, therefore path must be a file"); + .and_then(|n| n.to_str()) + .expect("already opened, therefore path must be a file"); let disktree_len = hextree.len(); let pb = progress_group.add(progress::bar( disktree_file_name.to_string(), diff --git a/hexit/src/main.rs b/hexit/src/main.rs index 89098bb..9f6acdf 100644 --- a/hexit/src/main.rs +++ b/hexit/src/main.rs @@ -1,5 +1,6 @@ mod combine; mod lookup; +mod mask; mod options; mod progress; mod tesselate; diff --git a/hexit/src/mask.rs b/hexit/src/mask.rs new file mode 100644 index 0000000..5482226 --- /dev/null +++ b/hexit/src/mask.rs @@ -0,0 +1,16 @@ +use anyhow::Result; +use geo::GeometryCollection; +use geojson::{quick_collection, GeoJson}; +use std::{fs::File, path::Path}; + +pub fn open(maybe_path: Option<&Path>) -> Result> { + match maybe_path { + None => Ok(None), + Some(path) => { + let mask_file = File::open(path)?; + let mask_json = GeoJson::from_reader(mask_file)?; + let mask: GeometryCollection = quick_collection(&mask_json)?; + Ok(Some(mask)) + } + } +} diff --git a/hexit/src/options.rs b/hexit/src/options.rs index c74d766..d87ec9a 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -17,6 +17,12 @@ pub enum Cli { /// input file. #[derive(Debug, Clone, Args)] pub struct Tesselate { + /// Path GeoJSON mask. + /// + /// Any tiles which do not intersect the mask are ignored. + #[arg(short, long)] + pub mask: Option, + /// Reprocess height file even if corresponding output already /// exists. #[arg(short = 'O', long)] @@ -40,6 +46,12 @@ pub struct Tesselate { /// Combine previously tesselated files into a single #[derive(Debug, Clone, Args)] pub struct Combine { + /// Path GeoJSON mask. + /// + /// Any samples which do not intersect the mask are ignored. + #[arg(short, long)] + pub mask: Option, + #[arg(short, long, default_value_t = Resolution::Ten)] pub resolution: Resolution, diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index e7e99c1..392acc4 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -1,7 +1,8 @@ -use crate::{options::Tesselate, progress}; +use crate::{mask, options::Tesselate, progress}; use anyhow::Result; use byteorder::{LittleEndian as LE, WriteBytesExt}; use flate2::{write::GzEncoder, Compression}; +use geo::{GeometryCollection, Intersects}; use h3o::{ geom::{PolyfillConfig, Polygon, ToCells}, CellIndex, Resolution, @@ -18,19 +19,24 @@ use std::{ impl Tesselate { pub fn run(&self) -> Result<()> { let progress_group = MultiProgress::new(); - self.input - .par_iter() - .try_for_each(|height_file_path| self._run(height_file_path, &progress_group))?; + let mask = mask::open(self.mask.as_deref())?; + self.input.par_iter().try_for_each(|height_file_path| { + self._run(height_file_path, mask.as_ref(), &progress_group) + })?; Ok(()) } - fn _run(&self, height_file_path: &Path, progress_group: &MultiProgress) -> Result<()> { + fn _run( + &self, + height_file_path: &Path, + mask: Option<&GeometryCollection>, + progress_group: &MultiProgress, + ) -> Result<()> { let out_file_name = { let file_name = height_file_path .file_name() - .expect("we already parsed the tile, therefore path must be a file") - .to_str() - .expect("we already parsed the tile, therefore path must be a file"); + .and_then(|n| n.to_str()) + .expect("already opened, therefore path must be a file"); format!("{file_name}.res{}.h3tez", self.resolution) }; let out_file_path = self.out_dir.clone().join(&out_file_name); @@ -47,11 +53,17 @@ impl Tesselate { }; let tile = Tile::memmap(height_file_path)?; - let pb = progress_group.add(progress::bar(out_file_name, tile.len() as u64)); - let tmp_out_file = File::create(&out_file_tmp_path)?; - let tmp_out_wtr = GzEncoder::new(tmp_out_file, Compression::new(self.compression)); - self.polyfill_tile(&tile, &pb, BufWriter::new(tmp_out_wtr))?; - fs::rename(out_file_tmp_path, out_file_path)?; + let intersects = mask + .as_ref() + .map_or(true, |mask| mask.intersects(&tile.polygon())); + if intersects { + let pb = progress_group.add(progress::bar(out_file_name, tile.len() as u64)); + let tmp_out_file = File::create(&out_file_tmp_path)?; + let tmp_out_wtr = GzEncoder::new(tmp_out_file, Compression::new(self.compression)); + self.polyfill_tile(&tile, &pb, BufWriter::new(tmp_out_wtr))?; + fs::rename(out_file_tmp_path, out_file_path)?; + } + Ok(()) } From 973239548ffefc6015efc916dd3713f9e0f2f0c3 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 20:34:04 -0700 Subject: [PATCH 06/18] [hexit] use jemalloc --- hexit/Cargo.toml | 4 ++++ hexit/src/main.rs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml index bfbdad6..b60173e 100644 --- a/hexit/Cargo.toml +++ b/hexit/Cargo.toml @@ -18,3 +18,7 @@ itertools = "0.10" nasadem = { path = "../nasadem" } num-traits = { workspace = true } rayon = "1.8.0" + + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = "0.5" diff --git a/hexit/src/main.rs b/hexit/src/main.rs index 9f6acdf..ce183f5 100644 --- a/hexit/src/main.rs +++ b/hexit/src/main.rs @@ -8,6 +8,12 @@ mod tesselate; use anyhow::Result; use clap::Parser; use options::Cli; +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; fn main() -> Result<()> { let cli = Cli::parse(); From dbb214f33227201fe35605555aee8c8198a6aef8 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Tue, 7 Nov 2023 20:36:05 -0700 Subject: [PATCH 07/18] Fix lints --- hexit/src/combine.rs | 6 +++--- hexit/src/tesselate.rs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 9694a7b..aacfeca 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -6,7 +6,7 @@ use geo::{coord, GeometryCollection, Intersects}; use h3o::{CellIndex, LatLng}; use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap}; use indicatif::MultiProgress; -use std::{fs::File, io::BufReader, path::Path}; +use std::{ffi::OsStr, fs::File, io::BufReader, path::Path}; impl Combine { pub fn run(&self) -> Result<()> { @@ -33,7 +33,7 @@ impl Combine { let mut rdr = GzDecoder::new(tess_buf_rdr); let tess_file_name = tess_file_path .file_name() - .and_then(|n| n.to_str()) + .and_then(OsStr::to_str) .expect("already opened, therefore path must be a file"); let n_samples = rdr.read_u64::()?; @@ -73,7 +73,7 @@ impl Combine { let disktree_file_name = self .out .file_name() - .and_then(|n| n.to_str()) + .and_then(OsStr::to_str) .expect("already opened, therefore path must be a file"); let disktree_len = hextree.len(); let pb = progress_group.add(progress::bar( diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 392acc4..10496f1 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -11,6 +11,7 @@ use indicatif::{MultiProgress, ProgressBar}; use nasadem::{Sample, Tile}; use rayon::prelude::*; use std::{ + ffi::OsStr, fs::{self, File}, io::{BufWriter, Write}, path::Path, @@ -35,7 +36,7 @@ impl Tesselate { let out_file_name = { let file_name = height_file_path .file_name() - .and_then(|n| n.to_str()) + .and_then(OsStr::to_str) .expect("already opened, therefore path must be a file"); format!("{file_name}.res{}.h3tez", self.resolution) }; From 7825be4d26e893105c2e3d131f35d319849d4db9 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Fri, 10 Nov 2023 12:19:35 -0700 Subject: [PATCH 08/18] wip --- hexit/src/combine.rs | 34 +++++++----------- hexit/src/lookup.rs | 33 +++++++++++++++-- hexit/src/options.rs | 3 ++ hexit/src/tesselate.rs | 80 ++++++++++++++++++++++++++++-------------- 4 files changed, 99 insertions(+), 51 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index aacfeca..6e4846e 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,9 +1,7 @@ -use crate::{mask, options::Combine, progress}; +use crate::{options::Combine, progress}; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use flate2::bufread::GzDecoder; -use geo::{coord, GeometryCollection, Intersects}; -use h3o::{CellIndex, LatLng}; use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap}; use indicatif::MultiProgress; use std::{ffi::OsStr, fs::File, io::BufReader, path::Path}; @@ -13,9 +11,8 @@ impl Combine { assert!(!self.input.is_empty()); let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); let progress_group = MultiProgress::new(); - let mask = mask::open(self.mask.as_deref())?; for tess_file_path in &self.input { - Self::read_tessellation(tess_file_path, &progress_group, mask.as_ref(), &mut hextree)?; + Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; } self.write_disktree(&hextree, &progress_group)?; self.verify_disktree(&hextree, &progress_group)?; @@ -25,7 +22,6 @@ impl Combine { fn read_tessellation( tess_file_path: &Path, progress_group: &MultiProgress, - mask: Option<&GeometryCollection>, hextree: &mut HexTreeMap, ) -> Result<()> { let tess_file = File::open(tess_file_path)?; @@ -39,21 +35,10 @@ impl Combine { let n_samples = rdr.read_u64::()?; let pb = progress_group.add(progress::bar(tess_file_name.to_string(), n_samples)); for _sample_n in 0..n_samples { + let raw_cell = rdr.read_u64::()?; + let cell = hextree::Cell::from_raw(raw_cell)?; let elevation = rdr.read_i16::()?; - let n_cells = rdr.read_u16::()?; - for _cell_n in 0..n_cells { - let raw_cell = rdr.read_u64::()?; - let cell = hextree::Cell::from_raw(raw_cell)?; - let mask_contains_cell = mask.as_ref().map_or(true, |mask| { - let cell_index = CellIndex::try_from(cell.into_raw()).unwrap(); - let cell_center = LatLng::from(cell_index); - let coord = coord!(x: cell_center.lng(), y: cell_center.lat()); - mask.intersects(&coord) - }); - if mask_contains_cell { - hextree.insert(cell, elevation); - } - } + hextree.insert(cell, elevation); pb.inc(1); } assert!( @@ -77,7 +62,7 @@ impl Combine { .expect("already opened, therefore path must be a file"); let disktree_len = hextree.len(); let pb = progress_group.add(progress::bar( - disktree_file_name.to_string(), + format!("Writing {disktree_file_name}"), disktree_len as u64, )); hextree.to_disktree(disktree_file, |wtr, val| { @@ -98,8 +83,13 @@ impl Combine { } let mut disktree = DiskTree::open(&self.out)?; + let disktree_file_name = self + .out + .file_name() + .and_then(OsStr::to_str) + .expect("already opened, therefore path must be a file"); let pb = progress_group.add(progress::bar( - "Validating disktree".to_string(), + format!("Validating {disktree_file_name}"), hextree.len() as u64, )); let mut count = 0; diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index 91f6a73..3cb3f72 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -2,6 +2,7 @@ use crate::options::Lookup; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt}; use hextree::{disktree::DiskTree, Cell}; +use std::{fs::File, io::Write}; impl Lookup { pub fn run(&self) -> Result<()> { @@ -11,17 +12,45 @@ impl Lookup { .or_else(|_| u64::from_str_radix(&self.cell, 16))?; let cell = Cell::try_from(raw_cell)?; let mut disktree = DiskTree::open(&self.disktree)?; + + if self.iter { + Self::by_iter(cell, &mut disktree) + } else { + Self::by_get(cell, &mut disktree) + } + } + + fn by_get(cell: Cell, disktree: &mut DiskTree) -> Result<()> { let t0 = std::time::Instant::now(); match disktree.seek_to_cell(cell)? { None => (), Some((cell, rdr)) => { let t_seek = t0.elapsed(); - let val = rdr.read_i16::()?; + let elev = rdr.read_i16::()?; let t_tot = t0.elapsed(); - println!("{cell}: {val}"); + println!("{cell}: {elev}"); println!("{t_seek:?} {t_tot:?}"); } } Ok(()) } + + fn by_iter(_target_cell: Cell, disktree: &mut DiskTree) -> Result<()> { + fn read_elev(res: hextree::Result<(Cell, &mut File)>) -> Result> { + let (cell, rdr) = res?; + let mask = Cell::try_from(0x8126bffffffffff)?; + if cell.is_related_to(&mask) { + Ok(Some((cell, rdr.read_i16::()?))) + } else { + Ok(None) + } + } + let mut stderr = std::io::stderr().lock(); + for res in disktree.iter(read_elev)? { + if let Some((cell, elev)) = res? { + writeln!(&mut stderr, "{cell}: {elev}")?; + } + } + Ok(()) + } } diff --git a/hexit/src/options.rs b/hexit/src/options.rs index d87ec9a..4c94352 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -65,6 +65,9 @@ pub struct Combine { /// Lookup value for H3 cell in a disktree. #[derive(Debug, Clone, Args)] pub struct Lookup { + /// Iterate through the disktree instead of `get`ting the value. + #[arg(short, long)] + pub iter: bool, pub disktree: PathBuf, pub cell: String, } diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 10496f1..037c897 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -5,8 +5,9 @@ use flate2::{write::GzEncoder, Compression}; use geo::{GeometryCollection, Intersects}; use h3o::{ geom::{PolyfillConfig, Polygon, ToCells}, - CellIndex, Resolution, + Resolution, }; +use hextree::{compaction::EqCompactor, Cell, HexTreeMap}; use indicatif::{MultiProgress, ProgressBar}; use nasadem::{Sample, Tile}; use rayon::prelude::*; @@ -33,17 +34,19 @@ impl Tesselate { mask: Option<&GeometryCollection>, progress_group: &MultiProgress, ) -> Result<()> { - let out_file_name = { + let (in_file_name, out_file_name) = { let file_name = height_file_path .file_name() .and_then(OsStr::to_str) .expect("already opened, therefore path must be a file"); - format!("{file_name}.res{}.h3tez", self.resolution) + ( + file_name, + format!("{file_name}.res{}.h3tez", self.resolution), + ) }; let out_file_path = self.out_dir.clone().join(&out_file_name); if out_file_path.exists() && !self.overwrite { - // Exit early if we've already processed this input. return Ok(()); } @@ -54,46 +57,69 @@ impl Tesselate { }; let tile = Tile::memmap(height_file_path)?; - let intersects = mask - .as_ref() - .map_or(true, |mask| mask.intersects(&tile.polygon())); + let intersects = mask.as_ref().map_or(true, |mask| { + let polygon = tile.polygon(); + mask.intersects(&polygon) + }); if intersects { - let pb = progress_group.add(progress::bar(out_file_name, tile.len() as u64)); + let pb = progress_group.add(progress::bar( + format!("Tesselate {in_file_name}"), + tile.len() as u64, + )); + let hextree = self.tesselate_tile(&tile, &pb)?; let tmp_out_file = File::create(&out_file_tmp_path)?; let tmp_out_wtr = GzEncoder::new(tmp_out_file, Compression::new(self.compression)); - self.polyfill_tile(&tile, &pb, BufWriter::new(tmp_out_wtr))?; + let wtr = BufWriter::new(tmp_out_wtr); + let pb = progress_group.add(progress::bar( + format!("Write {out_file_name}"), + tile.len() as u64, + )); + Self::write_to_disk(&hextree, &pb, wtr)?; fs::rename(out_file_tmp_path, out_file_path)?; } Ok(()) } - fn polyfill_tile( + fn tesselate_tile( &self, tile: &Tile, progress_bar: &ProgressBar, - mut out: impl Write, - ) -> Result<()> { - out.write_u64::(tile.len() as u64)?; + ) -> Result> { + let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); for sample in tile.iter() { - let (elev, hexes) = polyfill_sample(&sample, self.resolution)?; - out.write_i16::(elev)?; - out.write_u16::(u16::try_from(hexes.len())?)?; + let (elev, hexes) = Self::tesselate_sample(&sample, self.resolution)?; for hex in hexes { - out.write_u64::(hex)?; + hextree.insert(Cell::try_from(hex)?, elev); } progress_bar.inc(1); } - Ok(()) + Ok(hextree) } -} -fn polyfill_sample(sample: &Sample, resolution: Resolution) -> Result<(i16, Vec)> { - let elevation = sample.elevation(); - let polygon = Polygon::from_degrees(sample.polygon())?; - let cell_iter = polygon.to_cells(PolyfillConfig::new(resolution)); - let mut cells: Vec = CellIndex::compact(cell_iter)?.map(u64::from).collect(); - cells.sort_unstable(); - cells.dedup(); - Ok((elevation, cells)) + fn tesselate_sample(sample: &Sample, resolution: Resolution) -> Result<(i16, Vec)> { + let elevation = sample.elevation(); + let polygon = Polygon::from_degrees(sample.polygon())?; + let mut cells: Vec = polygon + .to_cells(PolyfillConfig::new(resolution)) + .map(u64::from) + .collect(); + cells.sort_unstable(); + cells.dedup(); + Ok((elevation, cells)) + } + + fn write_to_disk( + hextree: &HexTreeMap, + progress_bar: &ProgressBar, + mut out: impl Write, + ) -> Result<()> { + out.write_u64::(hextree.len() as u64)?; + for (cell, elev) in hextree.iter() { + out.write_u64::(cell.into_raw())?; + out.write_i16::(*elev)?; + progress_bar.inc(1); + } + Ok(()) + } } From 9ff40c6d33e0422165a657c0a0587bfe9116f353 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Mon, 20 Nov 2023 16:13:27 -0700 Subject: [PATCH 09/18] wip --- .github/workflows/rust.yml | 2 +- Cargo.toml | 1 + hexit/Cargo.toml | 3 +- hexit/src/combine.rs | 105 +++++++++++++++++++++++-------------- hexit/src/elevation.rs | 81 ++++++++++++++++++++++++++++ hexit/src/json.rs | 78 +++++++++++++++++++++++++++ hexit/src/main.rs | 3 ++ hexit/src/options.rs | 23 +++++++- hexit/src/tesselate.rs | 14 ++--- 9 files changed, 257 insertions(+), 53 deletions(-) create mode 100644 hexit/src/elevation.rs create mode 100644 hexit/src/json.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2bdbfd0..cf7251f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -34,4 +34,4 @@ jobs: run: cargo fmt -- --check - name: Hygiene | Clippy - run: cargo clippy --all-targets --all-features -- -Dwarnings -Dclippy::all -Dclippy::pedantic -Aclippy::module_name_repetitions -Aclippy::missing_panics_doc -Aclippy::missing_errors_doc -Aclippy::must_use_candidate -Aclippy::similar_names + run: cargo clippy --all-targets --all-features -- -Dwarnings -Dclippy::all -Dclippy::pedantic -Aclippy::missing_errors_doc -Aclippy::missing_panics_doc -Aclippy::module_name_repetitions -Aclippy::must_use_candidate -Aclippy::similar_names -Aclippy::unreadable_literal diff --git a/Cargo.toml b/Cargo.toml index 804ce5d..67e4e7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ log = "0.4.20" memmap2 = "0.7.1" num-traits = "0.2.16" serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1.0.48" # We want meaninful stack traces when profiling/debugging diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml index b60173e..e152742 100644 --- a/hexit/Cargo.toml +++ b/hexit/Cargo.toml @@ -18,7 +18,8 @@ itertools = "0.10" nasadem = { path = "../nasadem" } num-traits = { workspace = true } rayon = "1.8.0" - +serde = { workspace = true } +serde_json = { workspace = true } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.5" diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 6e4846e..4bee834 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,28 +1,59 @@ -use crate::{options::Combine, progress}; +use crate::{ + elevation::{Elevation, ReducedElevation}, + options::Combine, + progress, +}; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use flate2::bufread::GzDecoder; -use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap}; +use hextree::{compaction::Compactor, Cell, HexTreeMap}; use indicatif::MultiProgress; use std::{ffi::OsStr, fs::File, io::BufReader, path::Path}; +struct ReductionCompactor { + target_resolution: u8, + source_resolution: u8, +} + +impl Compactor for ReductionCompactor { + fn compact(&mut self, cell: Cell, children: [Option<&Elevation>; 7]) -> Option { + if cell.res() < self.target_resolution { + None + } else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = + children + { + Some(Elevation::concat( + self.source_resolution, + cell.res(), + &[*v0, *v1, *v2, *v3, *v4, *v5, *v6], + )) + } else { + None + } + } +} + impl Combine { pub fn run(&self) -> Result<()> { assert!(!self.input.is_empty()); - let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); + let mut hextree: HexTreeMap = + HexTreeMap::with_compactor(ReductionCompactor { + source_resolution: self.source_resolution as u8, + target_resolution: self.target_resolution as u8, + }); let progress_group = MultiProgress::new(); for tess_file_path in &self.input { Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; } + let hextree = self.reduce_hextree(&hextree, &progress_group); self.write_disktree(&hextree, &progress_group)?; - self.verify_disktree(&hextree, &progress_group)?; Ok(()) } fn read_tessellation( tess_file_path: &Path, progress_group: &MultiProgress, - hextree: &mut HexTreeMap, + hextree: &mut HexTreeMap, ) -> Result<()> { let tess_file = File::open(tess_file_path)?; let tess_buf_rdr = BufReader::new(tess_file); @@ -38,7 +69,7 @@ impl Combine { let raw_cell = rdr.read_u64::()?; let cell = hextree::Cell::from_raw(raw_cell)?; let elevation = rdr.read_i16::()?; - hextree.insert(cell, elevation); + hextree.insert(cell, Elevation::Plain(elevation)); pb.inc(1); } assert!( @@ -49,9 +80,32 @@ impl Combine { Ok(()) } + fn reduce_hextree( + &self, + hextree: &HexTreeMap, + _progress_group: &MultiProgress, + ) -> HexTreeMap { + let mut reduced_hextree = HexTreeMap::new(); + let max_child_cnt = + 7_usize.pow(self.source_resolution as u32 - self.target_resolution as u32); + for (cell, elev) in hextree.iter() { + match elev { + Elevation::Intermediate(intermediate) + if cell.res() == self.target_resolution as u8 => + { + assert_eq!(intermediate.n, max_child_cnt); + let reduction = intermediate.reduce(); + reduced_hextree.insert(cell, reduction); + } + _ => {} + }; + } + reduced_hextree + } + fn write_disktree( &self, - hextree: &HexTreeMap, + hextree: &HexTreeMap, progress_group: &MultiProgress, ) -> Result<()> { let disktree_file = File::create(&self.out)?; @@ -65,41 +119,12 @@ impl Combine { format!("Writing {disktree_file_name}"), disktree_len as u64, )); - hextree.to_disktree(disktree_file, |wtr, val| { + hextree.to_disktree(disktree_file, |wtr, ReducedElevation { min, avg, max }| { pb.inc(1); - wtr.write_i16::(*val) + wtr.write_i16::(*min) + .and_then(|()| wtr.write_i16::(*avg)) + .and_then(|()| wtr.write_i16::(*max)) })?; Ok(()) } - - fn verify_disktree( - &self, - hextree: &HexTreeMap, - progress_group: &MultiProgress, - ) -> Result<()> { - fn value_reader(res: hextree::Result<(Cell, &mut File)>) -> Result<(Cell, i16)> { - let (cell, rdr) = res?; - Ok(rdr.read_i16::().map(|val| (cell, val))?) - } - - let mut disktree = DiskTree::open(&self.out)?; - let disktree_file_name = self - .out - .file_name() - .and_then(OsStr::to_str) - .expect("already opened, therefore path must be a file"); - let pb = progress_group.add(progress::bar( - format!("Validating {disktree_file_name}"), - hextree.len() as u64, - )); - let mut count = 0; - for res in disktree.iter(value_reader)? { - let (cell, value) = res?; - assert_eq!(Some((cell, &value)), hextree.get(cell)); - pb.inc(1); - count += 1; - } - assert_eq!(hextree.len(), count); - Ok(()) - } } diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs new file mode 100644 index 0000000..3cbd2bc --- /dev/null +++ b/hexit/src/elevation.rs @@ -0,0 +1,81 @@ +use anyhow::Result; +use byteorder::{LittleEndian as LE, ReadBytesExt}; +use std::io::Read; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReducedElevation { + pub min: i16, + pub avg: i16, + pub max: i16, +} + +impl ReducedElevation { + pub fn from_reader(mut rdr: R) -> Result { + let mut buf = [0_u8; 3 * std::mem::size_of::()]; + rdr.read_exact(&mut buf)?; + let rdr = &mut &buf[..]; + let min = rdr.read_i16::()?; + let avg = rdr.read_i16::()?; + let max = rdr.read_i16::()?; + Ok(Self { min, avg, max }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IntermediateElevation { + pub min: i16, + pub sum: i32, + pub max: i16, + pub n: usize, +} + +impl IntermediateElevation { + pub fn reduce(&self) -> ReducedElevation { + let min = self.min; + let avg = i16::try_from(self.sum / i32::try_from(self.n).unwrap()).unwrap(); + let max = self.max; + assert!(min <= avg && avg <= max); + ReducedElevation { min, avg, max } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Elevation { + Plain(i16), + Intermediate(IntermediateElevation), +} + +impl Elevation { + pub fn concat(source_resolution: u8, this_resolution: u8, items: &[Self]) -> Self { + let mut new_min = i16::MAX; + let mut new_sum: i32 = 0; + let mut new_max = i16::MIN; + let mut new_n = 0_usize; + for item in items { + match item { + Elevation::Plain(elev) => { + let n = 7_usize.pow(u32::from(source_resolution - this_resolution - 1)); + assert_ne!(n, 0); + let sum = i32::from(*elev) * i32::try_from(n).unwrap(); + new_sum += sum; + new_min = i16::min(new_min, *elev); + new_max = i16::max(new_max, *elev); + new_n += n; + } + + Elevation::Intermediate(IntermediateElevation { min, sum, max, n }) => { + new_sum += *sum; + new_min = i16::min(new_min, *min); + new_max = i16::max(new_max, *max); + new_n += n; + } + } + } + Elevation::Intermediate(IntermediateElevation { + min: new_min, + sum: new_sum, + max: new_max, + n: new_n, + }) + } +} diff --git a/hexit/src/json.rs b/hexit/src/json.rs new file mode 100644 index 0000000..3c90faf --- /dev/null +++ b/hexit/src/json.rs @@ -0,0 +1,78 @@ +use crate::{elevation::ReducedElevation, mask, options::Json}; +use anyhow::Result; +use geo::geometry::GeometryCollection; +use h3o::{ + geom::{PolyfillConfig, ToCells}, + Resolution, +}; +use hextree::{disktree::DiskTree, Cell, HexTreeMap}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::fs::File; + +impl Json { + pub fn run(&self) -> Result<()> { + let mut disktree = DiskTree::open(&self.disktree)?; + let mask = mask::open(Some(&self.mask))?.unwrap(); + let target_cells = Self::polyfill_mask(mask, self.source_resolution)?; + let mut hextree = HexTreeMap::new(); + for h3idx in target_cells { + let cell = Cell::try_from(h3idx)?; + if let Some((cell, reduction)) = Self::get(cell, &mut disktree)? { + hextree.insert(cell, reduction); + } + } + let json = Self::gen_json(&hextree); + Self::output_json(&json)?; + Ok(()) + } + + fn polyfill_mask(mask: GeometryCollection, resolution: Resolution) -> Result> { + let polygon = h3o::geom::GeometryCollection::from_degrees(mask)?; + let mut cells: Vec = polygon + .to_cells(PolyfillConfig::new(resolution)) + .map(u64::from) + .collect(); + cells.sort_unstable(); + cells.dedup(); + Ok(cells) + } + + fn get(cell: Cell, disktree: &mut DiskTree) -> Result> { + match disktree.seek_to_cell(cell)? { + None => Ok(None), + Some((cell, rdr)) => { + let reduction = ReducedElevation::from_reader(rdr)?; + Ok(Some((cell, reduction))) + } + } + } + + fn gen_json(hextree: &HexTreeMap) -> Value { + #[derive(Serialize)] + struct JsonEntry { + h3_id: String, + min: i16, + avg: i16, + max: i16, + } + impl From<(Cell, &ReducedElevation)> for JsonEntry { + fn from((cell, reduction): (Cell, &ReducedElevation)) -> JsonEntry { + JsonEntry { + h3_id: cell.to_string(), + min: reduction.min, + avg: reduction.avg, + max: reduction.max, + } + } + } + let samples = hextree.iter().map(JsonEntry::from).collect::>(); + json!(samples) + } + + fn output_json(json: &Value) -> Result<()> { + let out = std::io::stdout(); + serde_json::to_writer(out, json)?; + Ok(()) + } +} diff --git a/hexit/src/main.rs b/hexit/src/main.rs index ce183f5..d075238 100644 --- a/hexit/src/main.rs +++ b/hexit/src/main.rs @@ -1,4 +1,6 @@ mod combine; +mod elevation; +mod json; mod lookup; mod mask; mod options; @@ -21,5 +23,6 @@ fn main() -> Result<()> { Cli::Tessellate(tesselate) => tesselate.run(), Cli::Combine(combine) => combine.run(), Cli::Lookup(lookup) => lookup.run(), + Cli::Json(json) => json.run(), } } diff --git a/hexit/src/options.rs b/hexit/src/options.rs index 4c94352..75c3505 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -11,6 +11,8 @@ pub enum Cli { Combine(Combine), Lookup(Lookup), + + Json(Json), } /// Generate a tessellated list of (cell, elevation) for each @@ -52,8 +54,11 @@ pub struct Combine { #[arg(short, long)] pub mask: Option, - #[arg(short, long, default_value_t = Resolution::Ten)] - pub resolution: Resolution, + #[arg(short, long)] + pub source_resolution: Resolution, + + #[arg(short, long)] + pub target_resolution: Resolution, #[arg(short, long)] pub out: PathBuf, @@ -71,3 +76,17 @@ pub struct Lookup { pub disktree: PathBuf, pub cell: String, } + +/// Output kepler.gl compatible JSON within the given mask. +#[derive(Debug, Clone, Args)] +pub struct Json { + #[arg(short, long)] + pub source_resolution: Resolution, + + /// Path GeoJSON mask. + /// + /// Any samples which do not intersect the mask are ignored. + pub mask: PathBuf, + + pub disktree: PathBuf, +} diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 037c897..4f33372 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -7,7 +7,7 @@ use h3o::{ geom::{PolyfillConfig, Polygon, ToCells}, Resolution, }; -use hextree::{compaction::EqCompactor, Cell, HexTreeMap}; +use hextree::{Cell, HexTreeMap}; use indicatif::{MultiProgress, ProgressBar}; use nasadem::{Sample, Tile}; use rayon::prelude::*; @@ -72,7 +72,7 @@ impl Tesselate { let wtr = BufWriter::new(tmp_out_wtr); let pb = progress_group.add(progress::bar( format!("Write {out_file_name}"), - tile.len() as u64, + hextree.len() as u64, )); Self::write_to_disk(&hextree, &pb, wtr)?; fs::rename(out_file_tmp_path, out_file_path)?; @@ -81,12 +81,8 @@ impl Tesselate { Ok(()) } - fn tesselate_tile( - &self, - tile: &Tile, - progress_bar: &ProgressBar, - ) -> Result> { - let mut hextree: HexTreeMap = HexTreeMap::with_compactor(EqCompactor); + fn tesselate_tile(&self, tile: &Tile, progress_bar: &ProgressBar) -> Result> { + let mut hextree: HexTreeMap = HexTreeMap::new(); for sample in tile.iter() { let (elev, hexes) = Self::tesselate_sample(&sample, self.resolution)?; for hex in hexes { @@ -110,7 +106,7 @@ impl Tesselate { } fn write_to_disk( - hextree: &HexTreeMap, + hextree: &HexTreeMap, progress_bar: &ProgressBar, mut out: impl Write, ) -> Result<()> { From 9f78d55e3160a13ce7e9f1bf044a80deb6528961 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Fri, 24 Nov 2023 13:25:02 -0700 Subject: [PATCH 10/18] Move compactor to elev mod --- hexit/src/combine.rs | 27 ++------------------------- hexit/src/elevation.rs | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 4bee834..3e2d2ea 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,38 +1,15 @@ use crate::{ - elevation::{Elevation, ReducedElevation}, + elevation::{Elevation, ReducedElevation, ReductionCompactor}, options::Combine, progress, }; use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use flate2::bufread::GzDecoder; -use hextree::{compaction::Compactor, Cell, HexTreeMap}; +use hextree::HexTreeMap; use indicatif::MultiProgress; use std::{ffi::OsStr, fs::File, io::BufReader, path::Path}; -struct ReductionCompactor { - target_resolution: u8, - source_resolution: u8, -} - -impl Compactor for ReductionCompactor { - fn compact(&mut self, cell: Cell, children: [Option<&Elevation>; 7]) -> Option { - if cell.res() < self.target_resolution { - None - } else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = - children - { - Some(Elevation::concat( - self.source_resolution, - cell.res(), - &[*v0, *v1, *v2, *v3, *v4, *v5, *v6], - )) - } else { - None - } - } -} - impl Combine { pub fn run(&self) -> Result<()> { assert!(!self.input.is_empty()); diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs index 3cbd2bc..11a86d8 100644 --- a/hexit/src/elevation.rs +++ b/hexit/src/elevation.rs @@ -1,5 +1,6 @@ use anyhow::Result; use byteorder::{LittleEndian as LE, ReadBytesExt}; +use hextree::{compaction::Compactor, Cell}; use std::io::Read; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -79,3 +80,26 @@ impl Elevation { }) } } + +pub struct ReductionCompactor { + pub target_resolution: u8, + pub source_resolution: u8, +} + +impl Compactor for ReductionCompactor { + fn compact(&mut self, cell: Cell, children: [Option<&Elevation>; 7]) -> Option { + if cell.res() < self.target_resolution { + None + } else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = + children + { + Some(Elevation::concat( + self.source_resolution, + cell.res(), + &[*v0, *v1, *v2, *v3, *v4, *v5, *v6], + )) + } else { + None + } + } +} From c3fcd179981405da62c2dd9c22b703b3fb16ec18 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Fri, 24 Nov 2023 13:39:04 -0700 Subject: [PATCH 11/18] Simplify Elevation type --- hexit/src/combine.rs | 13 ++++---- hexit/src/elevation.rs | 68 ++++++++++++++++-------------------------- 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 3e2d2ea..671ea21 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -45,8 +45,9 @@ impl Combine { for _sample_n in 0..n_samples { let raw_cell = rdr.read_u64::()?; let cell = hextree::Cell::from_raw(raw_cell)?; - let elevation = rdr.read_i16::()?; - hextree.insert(cell, Elevation::Plain(elevation)); + let raw_elevation = rdr.read_i16::()?; + let elevation = Elevation::new(raw_elevation); + hextree.insert(cell, elevation); pb.inc(1); } assert!( @@ -67,11 +68,9 @@ impl Combine { 7_usize.pow(self.source_resolution as u32 - self.target_resolution as u32); for (cell, elev) in hextree.iter() { match elev { - Elevation::Intermediate(intermediate) - if cell.res() == self.target_resolution as u8 => - { - assert_eq!(intermediate.n, max_child_cnt); - let reduction = intermediate.reduce(); + elevation if cell.res() == self.target_resolution as u8 => { + assert_eq!(elevation.n, max_child_cnt); + let reduction = elevation.reduce(); reduced_hextree.insert(cell, reduction); } _ => {} diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs index 11a86d8..027cd36 100644 --- a/hexit/src/elevation.rs +++ b/hexit/src/elevation.rs @@ -23,61 +23,49 @@ impl ReducedElevation { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct IntermediateElevation { +pub struct Elevation { pub min: i16, pub sum: i32, pub max: i16, pub n: usize, } -impl IntermediateElevation { +impl Elevation { + pub fn new(raw: i16) -> Elevation { + Elevation { + min: raw, + sum: i32::from(raw), + max: raw, + n: 1, + } + } + pub fn reduce(&self) -> ReducedElevation { let min = self.min; let avg = i16::try_from(self.sum / i32::try_from(self.n).unwrap()).unwrap(); let max = self.max; + assert_ne!(min, i16::MIN); + assert_ne!(min, i16::MAX); + assert_ne!(max, i16::MIN); + assert_ne!(max, i16::MAX); assert!(min <= avg && avg <= max); ReducedElevation { min, avg, max } } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Elevation { - Plain(i16), - Intermediate(IntermediateElevation), -} - impl Elevation { - pub fn concat(source_resolution: u8, this_resolution: u8, items: &[Self]) -> Self { - let mut new_min = i16::MAX; - let mut new_sum: i32 = 0; - let mut new_max = i16::MIN; - let mut new_n = 0_usize; + pub fn concat(items: &[Self]) -> Self { + let mut min = i16::MAX; + let mut sum: i32 = 0; + let mut max = i16::MIN; + let mut n = 0_usize; for item in items { - match item { - Elevation::Plain(elev) => { - let n = 7_usize.pow(u32::from(source_resolution - this_resolution - 1)); - assert_ne!(n, 0); - let sum = i32::from(*elev) * i32::try_from(n).unwrap(); - new_sum += sum; - new_min = i16::min(new_min, *elev); - new_max = i16::max(new_max, *elev); - new_n += n; - } - - Elevation::Intermediate(IntermediateElevation { min, sum, max, n }) => { - new_sum += *sum; - new_min = i16::min(new_min, *min); - new_max = i16::max(new_max, *max); - new_n += n; - } - } + sum += item.sum; + min = i16::min(min, item.min); + max = i16::max(max, item.max); + n += item.n; } - Elevation::Intermediate(IntermediateElevation { - min: new_min, - sum: new_sum, - max: new_max, - n: new_n, - }) + Elevation { min, sum, max, n } } } @@ -93,11 +81,7 @@ impl Compactor for ReductionCompactor { } else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = children { - Some(Elevation::concat( - self.source_resolution, - cell.res(), - &[*v0, *v1, *v2, *v3, *v4, *v5, *v6], - )) + Some(Elevation::concat(&[*v0, *v1, *v2, *v3, *v4, *v5, *v6])) } else { None } From 61f67ab00f9a945bcd0663a78e3e99c986b949f5 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Fri, 24 Nov 2023 13:53:04 -0700 Subject: [PATCH 12/18] Add close-enough compactor --- hexit/src/combine.rs | 27 ++++++++------------------- hexit/src/elevation.rs | 37 +++++++++++++++++++++++++++++++++++++ hexit/src/lookup.rs | 39 +++++++++------------------------------ hexit/src/options.rs | 11 +---------- 4 files changed, 55 insertions(+), 59 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 671ea21..3d125f5 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,5 +1,5 @@ use crate::{ - elevation::{Elevation, ReducedElevation, ReductionCompactor}, + elevation::{CloseEnoughCompactor, Elevation, ReducedElevation}, options::Combine, progress, }; @@ -13,16 +13,15 @@ use std::{ffi::OsStr, fs::File, io::BufReader, path::Path}; impl Combine { pub fn run(&self) -> Result<()> { assert!(!self.input.is_empty()); - let mut hextree: HexTreeMap = - HexTreeMap::with_compactor(ReductionCompactor { - source_resolution: self.source_resolution as u8, - target_resolution: self.target_resolution as u8, + let mut hextree: HexTreeMap = + HexTreeMap::with_compactor(CloseEnoughCompactor { + tolerance: self.tolerance, }); let progress_group = MultiProgress::new(); for tess_file_path in &self.input { Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; } - let hextree = self.reduce_hextree(&hextree, &progress_group); + let hextree = Self::reduce_hextree(&hextree, &progress_group); self.write_disktree(&hextree, &progress_group)?; Ok(()) } @@ -30,7 +29,7 @@ impl Combine { fn read_tessellation( tess_file_path: &Path, progress_group: &MultiProgress, - hextree: &mut HexTreeMap, + hextree: &mut HexTreeMap, ) -> Result<()> { let tess_file = File::open(tess_file_path)?; let tess_buf_rdr = BufReader::new(tess_file); @@ -59,22 +58,12 @@ impl Combine { } fn reduce_hextree( - &self, - hextree: &HexTreeMap, + hextree: &HexTreeMap, _progress_group: &MultiProgress, ) -> HexTreeMap { let mut reduced_hextree = HexTreeMap::new(); - let max_child_cnt = - 7_usize.pow(self.source_resolution as u32 - self.target_resolution as u32); for (cell, elev) in hextree.iter() { - match elev { - elevation if cell.res() == self.target_resolution as u8 => { - assert_eq!(elevation.n, max_child_cnt); - let reduction = elevation.reduce(); - reduced_hextree.insert(cell, reduction); - } - _ => {} - }; + reduced_hextree.insert(cell, elev.reduce()); } reduced_hextree } diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs index 027cd36..052467b 100644 --- a/hexit/src/elevation.rs +++ b/hexit/src/elevation.rs @@ -87,3 +87,40 @@ impl Compactor for ReductionCompactor { } } } + +pub struct CloseEnoughCompactor { + // Maximum differance between min and max child elevations + // allowable for a cell to be coalesced. + pub tolerance: i16, +} + +impl Compactor for CloseEnoughCompactor { + fn compact(&mut self, _cell: Cell, children: [Option<&Elevation>; 7]) -> Option { + if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = children { + let mut n_min = i16::MAX; + let mut n_sum = 0; + let mut n_max = i16::MIN; + let mut n_n = 0; + for Elevation { min, sum, max, n } in [v0, v1, v2, v3, v4, v5, v6] { + n_min = i16::min(n_min, *min); + n_sum += sum; + n_max = i16::max(n_max, *max); + n_n += n; + } + let error = n_max - n_min; + assert!(error >= 0, "error can't be negative"); + if error <= self.tolerance { + Some(Elevation { + min: n_min, + sum: n_sum, + max: n_max, + n: n_n, + }) + } else { + None + } + } else { + None + } + } +} diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index 3cb3f72..7c980f7 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -1,8 +1,7 @@ -use crate::options::Lookup; +use crate::{elevation::ReducedElevation, options::Lookup}; use anyhow::Result; -use byteorder::{LittleEndian as LE, ReadBytesExt}; use hextree::{disktree::DiskTree, Cell}; -use std::{fs::File, io::Write}; +use std::fs::File; impl Lookup { pub fn run(&self) -> Result<()> { @@ -13,11 +12,7 @@ impl Lookup { let cell = Cell::try_from(raw_cell)?; let mut disktree = DiskTree::open(&self.disktree)?; - if self.iter { - Self::by_iter(cell, &mut disktree) - } else { - Self::by_get(cell, &mut disktree) - } + Self::by_get(cell, &mut disktree) } fn by_get(cell: Cell, disktree: &mut DiskTree) -> Result<()> { @@ -26,29 +21,13 @@ impl Lookup { None => (), Some((cell, rdr)) => { let t_seek = t0.elapsed(); - let elev = rdr.read_i16::()?; + let ReducedElevation { min, avg, max } = ReducedElevation::from_reader(rdr)?; let t_tot = t0.elapsed(); - println!("{cell}: {elev}"); - println!("{t_seek:?} {t_tot:?}"); - } - } - Ok(()) - } - - fn by_iter(_target_cell: Cell, disktree: &mut DiskTree) -> Result<()> { - fn read_elev(res: hextree::Result<(Cell, &mut File)>) -> Result> { - let (cell, rdr) = res?; - let mask = Cell::try_from(0x8126bffffffffff)?; - if cell.is_related_to(&mask) { - Ok(Some((cell, rdr.read_i16::()?))) - } else { - Ok(None) - } - } - let mut stderr = std::io::stderr().lock(); - for res in disktree.iter(read_elev)? { - if let Some((cell, elev)) = res? { - writeln!(&mut stderr, "{cell}: {elev}")?; + println!("cell: {cell} (res {})", cell.res()); + println!("min: {min}"); + println!("avg: {avg}"); + println!("max: {max}"); + println!("seek: {t_seek:?}"); } } Ok(()) diff --git a/hexit/src/options.rs b/hexit/src/options.rs index 75c3505..5059b4d 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -48,17 +48,8 @@ pub struct Tesselate { /// Combine previously tesselated files into a single #[derive(Debug, Clone, Args)] pub struct Combine { - /// Path GeoJSON mask. - /// - /// Any samples which do not intersect the mask are ignored. - #[arg(short, long)] - pub mask: Option, - - #[arg(short, long)] - pub source_resolution: Resolution, - #[arg(short, long)] - pub target_resolution: Resolution, + pub tolerance: i16, #[arg(short, long)] pub out: PathBuf, From 7c12bc958ee9dfc6408dcbd4c8d5e70216fb6a6d Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Sat, 25 Nov 2023 10:41:39 -0700 Subject: [PATCH 13/18] Cleanup json generator --- hexit/src/json.rs | 2 +- hexit/src/lookup.rs | 1 - hexit/src/options.rs | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hexit/src/json.rs b/hexit/src/json.rs index 3c90faf..f4ebef1 100644 --- a/hexit/src/json.rs +++ b/hexit/src/json.rs @@ -14,7 +14,7 @@ impl Json { pub fn run(&self) -> Result<()> { let mut disktree = DiskTree::open(&self.disktree)?; let mask = mask::open(Some(&self.mask))?.unwrap(); - let target_cells = Self::polyfill_mask(mask, self.source_resolution)?; + let target_cells = Self::polyfill_mask(mask, self.resolution)?; let mut hextree = HexTreeMap::new(); for h3idx in target_cells { let cell = Cell::try_from(h3idx)?; diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index 7c980f7..54c161b 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -22,7 +22,6 @@ impl Lookup { Some((cell, rdr)) => { let t_seek = t0.elapsed(); let ReducedElevation { min, avg, max } = ReducedElevation::from_reader(rdr)?; - let t_tot = t0.elapsed(); println!("cell: {cell} (res {})", cell.res()); println!("min: {min}"); println!("avg: {avg}"); diff --git a/hexit/src/options.rs b/hexit/src/options.rs index 5059b4d..29d6157 100644 --- a/hexit/src/options.rs +++ b/hexit/src/options.rs @@ -71,8 +71,9 @@ pub struct Lookup { /// Output kepler.gl compatible JSON within the given mask. #[derive(Debug, Clone, Args)] pub struct Json { + /// Source resolution. #[arg(short, long)] - pub source_resolution: Resolution, + pub resolution: Resolution, /// Path GeoJSON mask. /// From 7c5d49ce04a10a23b0810f212ac376d75f2f68b8 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Sun, 26 Nov 2023 14:19:15 -0700 Subject: [PATCH 14/18] Output raw elevation data --- hexit/src/combine.rs | 24 +++---------- hexit/src/elevation.rs | 77 +++++++++++++++++++++--------------------- hexit/src/json.rs | 36 ++++++++++++-------- hexit/src/lookup.rs | 6 ++-- 4 files changed, 69 insertions(+), 74 deletions(-) diff --git a/hexit/src/combine.rs b/hexit/src/combine.rs index 3d125f5..99ec4b2 100644 --- a/hexit/src/combine.rs +++ b/hexit/src/combine.rs @@ -1,10 +1,10 @@ use crate::{ - elevation::{CloseEnoughCompactor, Elevation, ReducedElevation}, + elevation::{CloseEnoughCompactor, Elevation}, options::Combine, progress, }; use anyhow::Result; -use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; +use byteorder::{LittleEndian as LE, ReadBytesExt}; use flate2::bufread::GzDecoder; use hextree::HexTreeMap; use indicatif::MultiProgress; @@ -21,7 +21,6 @@ impl Combine { for tess_file_path in &self.input { Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?; } - let hextree = Self::reduce_hextree(&hextree, &progress_group); self.write_disktree(&hextree, &progress_group)?; Ok(()) } @@ -57,20 +56,9 @@ impl Combine { Ok(()) } - fn reduce_hextree( - hextree: &HexTreeMap, - _progress_group: &MultiProgress, - ) -> HexTreeMap { - let mut reduced_hextree = HexTreeMap::new(); - for (cell, elev) in hextree.iter() { - reduced_hextree.insert(cell, elev.reduce()); - } - reduced_hextree - } - fn write_disktree( &self, - hextree: &HexTreeMap, + hextree: &HexTreeMap, progress_group: &MultiProgress, ) -> Result<()> { let disktree_file = File::create(&self.out)?; @@ -84,11 +72,9 @@ impl Combine { format!("Writing {disktree_file_name}"), disktree_len as u64, )); - hextree.to_disktree(disktree_file, |wtr, ReducedElevation { min, avg, max }| { + hextree.to_disktree(disktree_file, |wtr, elev| { pb.inc(1); - wtr.write_i16::(*min) - .and_then(|()| wtr.write_i16::(*avg)) - .and_then(|()| wtr.write_i16::(*max)) + elev.to_writer(wtr) })?; Ok(()) } diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs index 052467b..71756f3 100644 --- a/hexit/src/elevation.rs +++ b/hexit/src/elevation.rs @@ -1,36 +1,22 @@ -use anyhow::Result; -use byteorder::{LittleEndian as LE, ReadBytesExt}; +use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt}; use hextree::{compaction::Compactor, Cell}; -use std::io::Read; +use std::{ + io::{Read, Write}, + mem::size_of, +}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ReducedElevation { - pub min: i16, - pub avg: i16, - pub max: i16, -} - -impl ReducedElevation { - pub fn from_reader(mut rdr: R) -> Result { - let mut buf = [0_u8; 3 * std::mem::size_of::()]; - rdr.read_exact(&mut buf)?; - let rdr = &mut &buf[..]; - let min = rdr.read_i16::()?; - let avg = rdr.read_i16::()?; - let max = rdr.read_i16::()?; - Ok(Self { min, avg, max }) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Elevation { pub min: i16, - pub sum: i32, pub max: i16, - pub n: usize, + pub sum: i32, + pub n: i32, } impl Elevation { + const BUF_LEN: usize = + size_of::() + size_of::() + size_of::() + size_of::(); + pub fn new(raw: i16) -> Elevation { Elevation { min: raw, @@ -40,32 +26,45 @@ impl Elevation { } } - pub fn reduce(&self) -> ReducedElevation { - let min = self.min; - let avg = i16::try_from(self.sum / i32::try_from(self.n).unwrap()).unwrap(); - let max = self.max; - assert_ne!(min, i16::MIN); - assert_ne!(min, i16::MAX); - assert_ne!(max, i16::MIN); - assert_ne!(max, i16::MAX); - assert!(min <= avg && avg <= max); - ReducedElevation { min, avg, max } + pub fn from_reader(mut rdr: R) -> std::io::Result { + debug_assert_eq!(Self::BUF_LEN, size_of::()); + let mut buf = [0_u8; Self::BUF_LEN]; + rdr.read_exact(&mut buf)?; + let rdr = &mut &buf[..]; + let min = rdr.read_i16::()?; + let max = rdr.read_i16::()?; + let sum = rdr.read_i32::()?; + let n = rdr.read_i32::()?; + Ok(Self { min, max, sum, n }) + } + + pub fn to_writer(&self, mut wtr: W) -> std::io::Result<()> { + assert_eq!(Self::BUF_LEN, size_of::()); + let mut buf = [0_u8; Self::BUF_LEN]; + { + let mut buf_wtr = &mut buf[..]; + buf_wtr.write_i16::(self.min)?; + buf_wtr.write_i16::(self.max)?; + buf_wtr.write_i32::(self.sum)?; + buf_wtr.write_i32::(self.n)?; + } + wtr.write_all(&buf) } } impl Elevation { - pub fn concat(items: &[Self]) -> Self { + pub fn concat(items: &[&Self]) -> Self { let mut min = i16::MAX; let mut sum: i32 = 0; let mut max = i16::MIN; - let mut n = 0_usize; + let mut n = 0_i32; for item in items { sum += item.sum; min = i16::min(min, item.min); max = i16::max(max, item.max); n += item.n; } - Elevation { min, sum, max, n } + Elevation { min, max, sum, n } } } @@ -81,7 +80,7 @@ impl Compactor for ReductionCompactor { } else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] = children { - Some(Elevation::concat(&[*v0, *v1, *v2, *v3, *v4, *v5, *v6])) + Some(Elevation::concat(&[v0, v1, v2, v3, v4, v5, v6])) } else { None } diff --git a/hexit/src/json.rs b/hexit/src/json.rs index f4ebef1..4e05bc6 100644 --- a/hexit/src/json.rs +++ b/hexit/src/json.rs @@ -1,4 +1,4 @@ -use crate::{elevation::ReducedElevation, mask, options::Json}; +use crate::{elevation::Elevation, mask, options::Json}; use anyhow::Result; use geo::geometry::GeometryCollection; use h3o::{ @@ -7,7 +7,7 @@ use h3o::{ }; use hextree::{disktree::DiskTree, Cell, HexTreeMap}; use serde::Serialize; -use serde_json::{json, Value}; +use serde_json::Value; use std::fs::File; impl Json { @@ -22,7 +22,7 @@ impl Json { hextree.insert(cell, reduction); } } - let json = Self::gen_json(&hextree); + let json = Self::gen_json(&hextree)?; Self::output_json(&json)?; Ok(()) } @@ -38,40 +38,48 @@ impl Json { Ok(cells) } - fn get(cell: Cell, disktree: &mut DiskTree) -> Result> { + fn get(cell: Cell, disktree: &mut DiskTree) -> Result> { match disktree.seek_to_cell(cell)? { None => Ok(None), Some((cell, rdr)) => { - let reduction = ReducedElevation::from_reader(rdr)?; + let reduction = Elevation::from_reader(rdr)?; Ok(Some((cell, reduction))) } } } - fn gen_json(hextree: &HexTreeMap) -> Value { + fn gen_json(hextree: &HexTreeMap) -> Result { #[derive(Serialize)] struct JsonEntry { h3_id: String, min: i16, avg: i16, + sum: i32, max: i16, + n: i32, } - impl From<(Cell, &ReducedElevation)> for JsonEntry { - fn from((cell, reduction): (Cell, &ReducedElevation)) -> JsonEntry { + impl From<(Cell, &Elevation)> for JsonEntry { + fn from((cell, elev): (Cell, &Elevation)) -> JsonEntry { JsonEntry { + avg: i16::try_from(elev.sum / elev.n).unwrap(), h3_id: cell.to_string(), - min: reduction.min, - avg: reduction.avg, - max: reduction.max, + max: elev.max, + min: elev.min, + n: elev.n, + sum: elev.sum, } } } - let samples = hextree.iter().map(JsonEntry::from).collect::>(); - json!(samples) + let samples = hextree + .iter() + .map(JsonEntry::from) + .map(serde_json::to_value) + .collect::, _>>()?; + Ok(Value::Array(samples)) } fn output_json(json: &Value) -> Result<()> { - let out = std::io::stdout(); + let out = std::io::stdout().lock(); serde_json::to_writer(out, json)?; Ok(()) } diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index 54c161b..c3512e9 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -1,4 +1,4 @@ -use crate::{elevation::ReducedElevation, options::Lookup}; +use crate::{elevation::Elevation, options::Lookup}; use anyhow::Result; use hextree::{disktree::DiskTree, Cell}; use std::fs::File; @@ -21,11 +21,13 @@ impl Lookup { None => (), Some((cell, rdr)) => { let t_seek = t0.elapsed(); - let ReducedElevation { min, avg, max } = ReducedElevation::from_reader(rdr)?; + let Elevation { min, max, sum, n } = Elevation::from_reader(rdr)?; + let avg = sum / n; println!("cell: {cell} (res {})", cell.res()); println!("min: {min}"); println!("avg: {avg}"); println!("max: {max}"); + println!("n: {n}"); println!("seek: {t_seek:?}"); } } From 215d9d1bd68eb84836067d2b5a014fb234ad0219 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Sat, 30 Dec 2023 20:34:40 +0000 Subject: [PATCH 15/18] [hexit] reduce indicatif bottleneck --- hexit/src/tesselate.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 4f33372..71bbe7b 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -8,7 +8,7 @@ use h3o::{ Resolution, }; use hextree::{Cell, HexTreeMap}; -use indicatif::{MultiProgress, ProgressBar}; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget}; use nasadem::{Sample, Tile}; use rayon::prelude::*; use std::{ @@ -20,7 +20,7 @@ use std::{ impl Tesselate { pub fn run(&self) -> Result<()> { - let progress_group = MultiProgress::new(); + let progress_group = MultiProgress::with_draw_target(ProgressDrawTarget::stderr_with_hz(4)); let mask = mask::open(self.mask.as_deref())?; self.input.par_iter().try_for_each(|height_file_path| { self._run(height_file_path, mask.as_ref(), &progress_group) From c27c7cb24084978cfb144aa809d6797421992426 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Fri, 5 Jan 2024 10:47:35 -0700 Subject: [PATCH 16/18] Use latest disktree api --- hexit/src/json.rs | 15 +++++++-------- hexit/src/lookup.rs | 13 ++++++------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/hexit/src/json.rs b/hexit/src/json.rs index 4e05bc6..b09f890 100644 --- a/hexit/src/json.rs +++ b/hexit/src/json.rs @@ -5,20 +5,19 @@ use h3o::{ geom::{PolyfillConfig, ToCells}, Resolution, }; -use hextree::{disktree::DiskTree, Cell, HexTreeMap}; +use hextree::{disktree::DiskTreeMap, memmap::Mmap, Cell, HexTreeMap}; use serde::Serialize; use serde_json::Value; -use std::fs::File; impl Json { pub fn run(&self) -> Result<()> { - let mut disktree = DiskTree::open(&self.disktree)?; + let disktree = DiskTreeMap::open(&self.disktree)?; let mask = mask::open(Some(&self.mask))?.unwrap(); let target_cells = Self::polyfill_mask(mask, self.resolution)?; let mut hextree = HexTreeMap::new(); for h3idx in target_cells { let cell = Cell::try_from(h3idx)?; - if let Some((cell, reduction)) = Self::get(cell, &mut disktree)? { + if let Some((cell, reduction)) = Self::get(cell, &disktree)? { hextree.insert(cell, reduction); } } @@ -38,11 +37,11 @@ impl Json { Ok(cells) } - fn get(cell: Cell, disktree: &mut DiskTree) -> Result> { - match disktree.seek_to_cell(cell)? { + fn get(cell: Cell, disktree: &DiskTreeMap) -> Result> { + match disktree.get(cell)? { None => Ok(None), - Some((cell, rdr)) => { - let reduction = Elevation::from_reader(rdr)?; + Some((cell, bytes)) => { + let reduction = Elevation::from_reader(&mut &bytes[..])?; Ok(Some((cell, reduction))) } } diff --git a/hexit/src/lookup.rs b/hexit/src/lookup.rs index c3512e9..8da23e3 100644 --- a/hexit/src/lookup.rs +++ b/hexit/src/lookup.rs @@ -1,7 +1,6 @@ use crate::{elevation::Elevation, options::Lookup}; use anyhow::Result; -use hextree::{disktree::DiskTree, Cell}; -use std::fs::File; +use hextree::{disktree::DiskTreeMap, memmap::Mmap, Cell}; impl Lookup { pub fn run(&self) -> Result<()> { @@ -10,18 +9,18 @@ impl Lookup { .parse::() .or_else(|_| u64::from_str_radix(&self.cell, 16))?; let cell = Cell::try_from(raw_cell)?; - let mut disktree = DiskTree::open(&self.disktree)?; + let mut disktree = DiskTreeMap::open(&self.disktree)?; Self::by_get(cell, &mut disktree) } - fn by_get(cell: Cell, disktree: &mut DiskTree) -> Result<()> { + fn by_get(cell: Cell, disktree: &mut DiskTreeMap) -> Result<()> { let t0 = std::time::Instant::now(); - match disktree.seek_to_cell(cell)? { + match disktree.get(cell)? { None => (), - Some((cell, rdr)) => { + Some((cell, bytes)) => { let t_seek = t0.elapsed(); - let Elevation { min, max, sum, n } = Elevation::from_reader(rdr)?; + let Elevation { min, max, sum, n } = Elevation::from_reader(&mut &bytes[..])?; let avg = sum / n; println!("cell: {cell} (res {})", cell.res()); println!("min: {min}"); From 667dc2c80986434ae20cef25d18a0a2b66ceaf9c Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Sun, 7 Jan 2024 01:12:27 +0000 Subject: [PATCH 17/18] Handle voids --- hexit/src/elevation.rs | 4 ++++ hexit/src/tesselate.rs | 1 + 2 files changed, 5 insertions(+) diff --git a/hexit/src/elevation.rs b/hexit/src/elevation.rs index 71756f3..d38b3d4 100644 --- a/hexit/src/elevation.rs +++ b/hexit/src/elevation.rs @@ -101,6 +101,10 @@ impl Compactor for CloseEnoughCompactor { let mut n_max = i16::MIN; let mut n_n = 0; for Elevation { min, sum, max, n } in [v0, v1, v2, v3, v4, v5, v6] { + // HACK: Ignore voids that snuck through. + if [min, max].contains(&&i16::MIN) { + continue; + } n_min = i16::min(n_min, *min); n_sum += sum; n_max = i16::max(n_max, *max); diff --git a/hexit/src/tesselate.rs b/hexit/src/tesselate.rs index 71bbe7b..22a71f3 100644 --- a/hexit/src/tesselate.rs +++ b/hexit/src/tesselate.rs @@ -84,6 +84,7 @@ impl Tesselate { fn tesselate_tile(&self, tile: &Tile, progress_bar: &ProgressBar) -> Result> { let mut hextree: HexTreeMap = HexTreeMap::new(); for sample in tile.iter() { + assert_ne!(sample.elevation(), i16::MIN); let (elev, hexes) = Self::tesselate_sample(&sample, self.resolution)?; for hex in hexes { hextree.insert(Cell::try_from(hex)?, elev); From 10e8e510d1fd2f53a11869b8b2220a0a4d535124 Mon Sep 17 00:00:00 2001 From: Jay Kickliter Date: Mon, 8 Jan 2024 17:22:09 +0000 Subject: [PATCH 18/18] wip --- hexit/Cargo.toml | 2 +- nasadem/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hexit/Cargo.toml b/hexit/Cargo.toml index e152742..9b391c0 100644 --- a/hexit/Cargo.toml +++ b/hexit/Cargo.toml @@ -12,7 +12,7 @@ flate2 = "1.0.28" geo = { workspace = true } geojson = "0.24.1" h3o = { version = "0.4.0", features = ["geo"] } -hextree = { git = "https://github.com/JayKickliter/HexTree.git", branch = "jsk/add-disk-repr", features = ["disktree"] } +hextree = { git = "https://github.com/JayKickliter/HexTree.git", branch = "main", features = ["disktree"] } indicatif = "0.17.7" itertools = "0.10" nasadem = { path = "../nasadem" } diff --git a/nasadem/src/lib.rs b/nasadem/src/lib.rs index 40dc1af..a269b6a 100755 --- a/nasadem/src/lib.rs +++ b/nasadem/src/lib.rs @@ -281,7 +281,7 @@ impl Tile { self.samples.get_unchecked(idx_1d) } - /// Returns and iterator over `self`'s grid squares. + /// Returns an iterator over `self`'s grid squares. pub fn iter(&self) -> impl Iterator> + '_ { (0..(self.dimensions.0 * self.dimensions.1)).map(|index| Sample { tile: self, index }) }