Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
JayKickliter committed Nov 21, 2023
1 parent 8b8897a commit 09ddc45
Show file tree
Hide file tree
Showing 9 changed files with 309 additions and 52 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ jobs:
run: cargo fmt -- --check

- name: Hygiene | Clippy
run: cargo clippy --all-targets --all-features -- -Dwarnings -Dclippy::all -Dclippy::pedantic -Aclippy::module_name_repetitions -Aclippy::missing_panics_doc -Aclippy::missing_errors_doc -Aclippy::must_use_candidate -Aclippy::similar_names
run: cargo clippy --all-targets --all-features -- -Dwarnings -Dclippy::all -Dclippy::pedantic -Aclippy::missing_errors_doc -Aclippy::missing_panics_doc -Aclippy::module_name_repetitions -Aclippy::must_use_candidate -Aclippy::similar_names -Aclippy::unreadable_literal
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ log = "0.4.20"
memmap2 = "0.7.1"
num-traits = "0.2.16"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "1.0.48"

# We want meaninful stack traces when profiling/debugging
Expand Down
3 changes: 2 additions & 1 deletion hexit/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ itertools = "0.10"
nasadem = { path = "../nasadem" }
num-traits = { workspace = true }
rayon = "1.8.0"

serde = { workspace = true }
serde_json = { workspace = true }

[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = "0.5"
156 changes: 117 additions & 39 deletions hexit/src/combine.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,59 @@
use crate::{options::Combine, progress};
use crate::{
elevation::{Elevation, ReducedElevation},
options::Combine,
progress,
};
use anyhow::Result;
use byteorder::{LittleEndian as LE, ReadBytesExt, WriteBytesExt};
use flate2::bufread::GzDecoder;
use hextree::{compaction::EqCompactor, disktree::DiskTree, Cell, HexTreeMap};
use hextree::{compaction::Compactor, Cell, HexTreeMap};
use indicatif::MultiProgress;
use std::{ffi::OsStr, fs::File, io::BufReader, path::Path};

struct ReductionCompactor {
target_resolution: u8,
source_resolution: u8,
}

impl Compactor<Elevation> for ReductionCompactor {
fn compact(&mut self, cell: Cell, children: [Option<&Elevation>; 7]) -> Option<Elevation> {
if cell.res() < self.target_resolution {
None
} else if let [Some(v0), Some(v1), Some(v2), Some(v3), Some(v4), Some(v5), Some(v6)] =
children
{
Some(Elevation::concat(
self.source_resolution,
cell.res(),
&[*v0, *v1, *v2, *v3, *v4, *v5, *v6],
))
} else {
None
}
}
}

impl Combine {
pub fn run(&self) -> Result<()> {
assert!(!self.input.is_empty());
let mut hextree: HexTreeMap<i16, EqCompactor> = HexTreeMap::with_compactor(EqCompactor);
let mut hextree: HexTreeMap<Elevation, ReductionCompactor> =
HexTreeMap::with_compactor(ReductionCompactor {
source_resolution: self.source_resolution as u8,
target_resolution: self.target_resolution as u8,
});
let progress_group = MultiProgress::new();
for tess_file_path in &self.input {
Self::read_tessellation(tess_file_path, &progress_group, &mut hextree)?;
}
let hextree = self.reduce_hextree(&hextree, &progress_group);
self.write_disktree(&hextree, &progress_group)?;
self.verify_disktree(&hextree, &progress_group)?;
Ok(())
}

fn read_tessellation(
tess_file_path: &Path,
progress_group: &MultiProgress,
hextree: &mut HexTreeMap<i16, EqCompactor>,
hextree: &mut HexTreeMap<Elevation, ReductionCompactor>,
) -> Result<()> {
let tess_file = File::open(tess_file_path)?;
let tess_buf_rdr = BufReader::new(tess_file);
Expand All @@ -38,7 +69,7 @@ impl Combine {
let raw_cell = rdr.read_u64::<LE>()?;
let cell = hextree::Cell::from_raw(raw_cell)?;
let elevation = rdr.read_i16::<LE>()?;
hextree.insert(cell, elevation);
hextree.insert(cell, Elevation::Plain(elevation));
pb.inc(1);
}
assert!(
Expand All @@ -49,9 +80,55 @@ impl Combine {
Ok(())
}

fn reduce_hextree(
&self,
hextree: &HexTreeMap<Elevation, ReductionCompactor>,
_progress_group: &MultiProgress,
) -> HexTreeMap<ReducedElevation> {
// let mut res_n_cells = HashSet::new();
let mut reduced_hextree = HexTreeMap::new();
let max_child_cnt =
7_usize.pow(self.source_resolution as u32 - self.target_resolution as u32);
for (cell, elev) in hextree.iter() {
match elev {
Elevation::Intermediate(intermediate)
if cell.res() == self.target_resolution as u8 =>
{
assert_eq!(intermediate.n, max_child_cnt);
let reduction = intermediate.reduce();
reduced_hextree.insert(cell, reduction);
}
_ => {
// res_n_cells.insert(
// cell.to_parent(self.target_resolution as u8)
// .unwrap()
// .into_raw(),
// );
}
};
}
// println!("[");
// for index in res_n_cells.into_iter() {
// let cell = Cell::try_from(index).unwrap();
// println!("{{\"h3_id\":\"{index:x}\",\"res\":{},\"val\":\"\"}},", cell.res());
// let subtree_iter = hextree.subtree_iter(cell);
// for (cell, val) in subtree_iter {
// println!(
// "{{\"h3_id\":\"{:x}\",\"res\":{},\"val\":\"{val:?}\"}},",
// cell.into_raw(),
// cell.res()
// );
// }
// }
// println!("]");
// assert!(false);
// assert!(!reduced_hextree.is_empty());
reduced_hextree
}

fn write_disktree(
&self,
hextree: &HexTreeMap<i16, EqCompactor>,
hextree: &HexTreeMap<ReducedElevation>,
progress_group: &MultiProgress,
) -> Result<()> {
let disktree_file = File::create(&self.out)?;
Expand All @@ -65,41 +142,42 @@ impl Combine {
format!("Writing {disktree_file_name}"),
disktree_len as u64,
));
hextree.to_disktree(disktree_file, |wtr, val| {
hextree.to_disktree(disktree_file, |wtr, ReducedElevation { min, avg, max }| {
pb.inc(1);
wtr.write_i16::<LE>(*val)
wtr.write_i16::<LE>(*min)
.and_then(|()| wtr.write_i16::<LE>(*avg))
.and_then(|()| wtr.write_i16::<LE>(*max))
})?;
Ok(())
}

fn verify_disktree(
&self,
hextree: &HexTreeMap<i16, EqCompactor>,
progress_group: &MultiProgress,
) -> Result<()> {
fn value_reader(res: hextree::Result<(Cell, &mut File)>) -> Result<(Cell, i16)> {
let (cell, rdr) = res?;
Ok(rdr.read_i16::<LE>().map(|val| (cell, val))?)
}

let mut disktree = DiskTree::open(&self.out)?;
let disktree_file_name = self
.out
.file_name()
.and_then(OsStr::to_str)
.expect("already opened, therefore path must be a file");
let pb = progress_group.add(progress::bar(
format!("Validating {disktree_file_name}"),
hextree.len() as u64,
));
let mut count = 0;
for res in disktree.iter(value_reader)? {
let (cell, value) = res?;
assert_eq!(Some((cell, &value)), hextree.get(cell));
pb.inc(1);
count += 1;
}
assert_eq!(hextree.len(), count);
Ok(())
}
// fn verify_disktree(
// &self,
// hextree: &HexTreeMap<Elevation, ReductionCompactor>,
// progress_group: &MultiProgress,
// ) -> Result<()> {
// fn value_reader(res: hextree::Result<(Cell, &mut File)>) -> Result<(Cell, i16)> {
// let (cell, rdr) = res?;
// Ok(rdr.read_i16::<LE>().map(|val| (cell, val))?)
// }
// let mut disktree = DiskTree::open(&self.out)?;
// let disktree_file_name = self
// .out
// .file_name()
// .and_then(OsStr::to_str)
// .expect("already opened, therefore path must be a file");
// let pb = progress_group.add(progress::bar(
// format!("Validating {disktree_file_name}"),
// hextree.len() as u64,
// ));
// let mut count = 0;
// for res in disktree.iter(value_reader)? {
// let (cell, value) = res?;
// assert_eq!(Some((cell, &value)), hextree.get(cell));
// pb.inc(1);
// count += 1;
// }
// assert_eq!(hextree.len(), count);
// Ok(())
// }
}
81 changes: 81 additions & 0 deletions hexit/src/elevation.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
use anyhow::Result;
use byteorder::{LittleEndian as LE, ReadBytesExt};
use std::io::Read;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ReducedElevation {
pub min: i16,
pub avg: i16,
pub max: i16,
}

impl ReducedElevation {
pub fn from_reader<R: Read>(mut rdr: R) -> Result<Self> {
let mut buf = [0_u8; 3 * std::mem::size_of::<i16>()];
rdr.read_exact(&mut buf)?;
let rdr = &mut &buf[..];
let min = rdr.read_i16::<LE>()?;
let avg = rdr.read_i16::<LE>()?;
let max = rdr.read_i16::<LE>()?;
Ok(Self { min, avg, max })
}
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct IntermediateElevation {
pub min: i16,
pub sum: i32,
pub max: i16,
pub n: usize,
}

impl IntermediateElevation {
pub fn reduce(&self) -> ReducedElevation {
let min = self.min;
let avg = i16::try_from(self.sum / i32::try_from(self.n).unwrap()).unwrap();
let max = self.max;
assert!(min <= avg && avg <= max);
ReducedElevation { min, avg, max }
}
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Elevation {
Plain(i16),
Intermediate(IntermediateElevation),
}

impl Elevation {
pub fn concat(source_resolution: u8, this_resolution: u8, items: &[Self]) -> Self {
let mut new_min = i16::MAX;
let mut new_sum: i32 = 0;
let mut new_max = i16::MIN;
let mut new_n = 0_usize;
for item in items {
match item {
Elevation::Plain(elev) => {
let n = 7_usize.pow(u32::from(source_resolution - this_resolution - 1));
assert_ne!(n, 0);
let sum = i32::from(*elev) * i32::try_from(n).unwrap();
new_sum += sum;
new_min = i16::min(new_min, *elev);
new_max = i16::max(new_max, *elev);
new_n += n;
}

Elevation::Intermediate(IntermediateElevation { min, sum, max, n }) => {
new_sum += *sum;
new_min = i16::min(new_min, *min);
new_max = i16::max(new_max, *max);
new_n += n;
}
}
}
Elevation::Intermediate(IntermediateElevation {
min: new_min,
sum: new_sum,
max: new_max,
n: new_n,
})
}
}
78 changes: 78 additions & 0 deletions hexit/src/json.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
use crate::{elevation::ReducedElevation, mask, options::Json};
use anyhow::Result;
use geo::geometry::GeometryCollection;
use h3o::{
geom::{PolyfillConfig, ToCells},
Resolution,
};
use hextree::{disktree::DiskTree, Cell, HexTreeMap};
use serde::Serialize;
use serde_json::{json, Value};
use std::fs::File;

impl Json {
pub fn run(&self) -> Result<()> {
let mut disktree = DiskTree::open(&self.disktree)?;
let mask = mask::open(Some(&self.mask))?.unwrap();
let target_cells = Self::polyfill_mask(mask, self.source_resolution)?;
let mut hextree = HexTreeMap::new();
for h3idx in target_cells {
let cell = Cell::try_from(h3idx)?;
if let Some((cell, reduction)) = Self::get(cell, &mut disktree)? {
hextree.insert(cell, reduction);
}
}
let json = Self::gen_json(&hextree);
Self::output_json(&json)?;
Ok(())
}

fn polyfill_mask(mask: GeometryCollection, resolution: Resolution) -> Result<Vec<u64>> {
let polygon = h3o::geom::GeometryCollection::from_degrees(mask)?;
let mut cells: Vec<u64> = polygon
.to_cells(PolyfillConfig::new(resolution))
.map(u64::from)
.collect();
cells.sort_unstable();
cells.dedup();
Ok(cells)
}

fn get(cell: Cell, disktree: &mut DiskTree<File>) -> Result<Option<(Cell, ReducedElevation)>> {
match disktree.seek_to_cell(cell)? {
None => Ok(None),
Some((cell, rdr)) => {
let reduction = ReducedElevation::from_reader(rdr)?;
Ok(Some((cell, reduction)))
}
}
}

fn gen_json(hextree: &HexTreeMap<ReducedElevation>) -> Value {
#[derive(Serialize)]
struct JsonEntry {
h3_id: String,
min: i16,
avg: i16,
max: i16,
}
impl From<(Cell, &ReducedElevation)> for JsonEntry {
fn from((cell, reduction): (Cell, &ReducedElevation)) -> JsonEntry {
JsonEntry {
h3_id: cell.to_string(),
min: reduction.min,
avg: reduction.avg,
max: reduction.max,
}
}
}
let samples = hextree.iter().map(JsonEntry::from).collect::<Vec<_>>();
json!(samples)
}

fn output_json(json: &Value) -> Result<()> {
let out = std::io::stdout();
serde_json::to_writer(out, json)?;
Ok(())
}
}
Loading

0 comments on commit 09ddc45

Please sign in to comment.