diff --git a/Cargo.lock b/Cargo.lock index 04d5c718..88df63a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "amethyst" version = "0.11.1" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_animation 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -90,7 +90,7 @@ dependencies = [ [[package]] name = "amethyst_animation" version = "0.6.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -109,7 +109,7 @@ dependencies = [ [[package]] name = "amethyst_assets" version = "0.7.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_error 0.1.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -139,7 +139,7 @@ dependencies = [ [[package]] name = "amethyst_audio" version = "0.6.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -155,7 +155,7 @@ dependencies = [ [[package]] name = "amethyst_config" version = "0.10.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "ron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -165,7 +165,7 @@ dependencies = [ [[package]] name = "amethyst_controls" version = "0.5.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -179,7 +179,7 @@ dependencies = [ [[package]] name = "amethyst_core" version = "0.6.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "alga 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "alga_derive 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -203,7 +203,7 @@ dependencies = [ [[package]] name = "amethyst_derive" version = "0.4.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", @@ -213,7 +213,7 @@ dependencies = [ [[package]] name = "amethyst_error" version = "0.1.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "backtrace 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -221,7 +221,7 @@ dependencies = [ [[package]] name = "amethyst_input" version = "0.7.1" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_config 0.10.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -237,7 +237,7 @@ dependencies = [ [[package]] name = "amethyst_locale" version = "0.5.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -250,7 +250,7 @@ dependencies = [ [[package]] name = "amethyst_network" version = "0.4.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_error 0.1.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -268,7 +268,7 @@ dependencies = [ [[package]] name = "amethyst_rendy" version = "0.1.2" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -287,7 +287,7 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "palette 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rendy 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ron 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", "shred 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "shred-derive 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -299,7 +299,7 @@ dependencies = [ [[package]] name = "amethyst_ui" version = "0.6.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_audio 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -321,7 +321,7 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "paste 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "ron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", "shred 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "amethyst_utils" version = "0.6.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_assets 0.7.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_controls 0.5.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -356,7 +356,7 @@ dependencies = [ [[package]] name = "amethyst_window" version = "0.1.0" -source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#4c5988d17abc3d7ec055d9debca373c3cd78b6dd" +source = "git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system#d48f2b844eff93dfa7f5fde5d55f26a5241036a4" dependencies = [ "amethyst_config 0.10.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", "amethyst_core 0.6.0 (git+https://github.com/kabergstrom/amethyst.git?branch=new-asset-system)", @@ -497,7 +497,7 @@ dependencies = [ [[package]] name = "atelier-importer" version = "0.1.0" -source = "git+https://github.com/amethyst/atelier-assets.git#1ba1d77ba4545088267c8955bdb998be5450faa5" +source = "git+https://github.com/amethyst/atelier-assets.git#6746c6027cdcc1e34920681ac516d4e380fe4e48" dependencies = [ "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "downcast 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -526,12 +526,12 @@ dependencies = [ [[package]] name = "atelier-loader" version = "0.1.0" -source = "git+https://github.com/amethyst/atelier-assets.git#1ba1d77ba4545088267c8955bdb998be5450faa5" +source = "git+https://github.com/amethyst/atelier-assets.git#6746c6027cdcc1e34920681ac516d4e380fe4e48" dependencies = [ "atelier-schema 0.1.0 (git+https://github.com/amethyst/atelier-assets.git)", - "capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "capnp-rpc 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ccl 4.5.3 (git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git)", + "capnp 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "capnp-rpc 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ccl 4.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-channel 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -566,10 +566,10 @@ dependencies = [ [[package]] name = "atelier-schema" version = "0.1.0" -source = "git+https://github.com/amethyst/atelier-assets.git#1ba1d77ba4545088267c8955bdb998be5450faa5" +source = "git+https://github.com/amethyst/atelier-assets.git#6746c6027cdcc1e34920681ac516d4e380fe4e48" dependencies = [ - "capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "capnpc 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", + "capnp 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -721,15 +721,6 @@ dependencies = [ "ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "capnp" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "capnp" version = "0.10.0" @@ -739,16 +730,6 @@ dependencies = [ "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "capnp-futures" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "capnp-futures" version = "0.10.0" @@ -759,17 +740,6 @@ dependencies = [ "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "capnp-rpc" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "capnp-futures 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "capnpc 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "capnp-rpc" version = "0.10.0" @@ -781,14 +751,6 @@ dependencies = [ "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "capnpc" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "capnpc" version = "0.10.1" @@ -814,20 +776,6 @@ name = "cc" version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "ccl" -version = "4.5.3" -source = "git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git#6b5150e9e032581e90301cb58c671b3f85d519bd" -dependencies = [ - "ccl-crossbeam-epoch 0.7.2 (git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git)", - "ccl_owning_ref 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "seahash 3.0.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ccl" version = "4.12.1" @@ -842,19 +790,6 @@ dependencies = [ "seahash 3.0.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ccl-crossbeam-epoch" -version = "0.7.2" -source = "git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git#6b5150e9e032581e90301cb58c671b3f85d519bd" -dependencies = [ - "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ccl-crossbeam-epoch" version = "0.7.3" @@ -2224,14 +2159,6 @@ dependencies = [ "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "lock_api" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "lock_api" version = "0.3.1" @@ -2861,16 +2788,6 @@ dependencies = [ "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "parking_lot" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "parking_lot" version = "0.9.0" @@ -2905,21 +2822,6 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "parking_lot_core" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "parking_lot_core" version = "0.6.1" @@ -5023,18 +4925,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7d64d04786e0f528460fc884753cf8dddcc466be308f6026f8e355c41a0e4101" "checksum capnp 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fadfee1d1134072232d629291d39205fa74cde71d2c645c09b7aa321c3dd6f4f" -"checksum capnp 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "50bc40e18a764f679fd13673b61c12d2c45c1581a2c6c949ffa424d9f58716c7" "checksum capnp-futures 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b5b2320da14df78d0f0c4622c8eeedf40c6680d6c2621d20192fb32784ce54" -"checksum capnp-futures 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80185c75c4184e3d1d6fa4705d1260effe2ea070a154b428a47899f941fc9255" "checksum capnp-rpc 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "568eecd404ea80e98d506b922be2de5e1013ac8f9b170242a53068affc79ddc8" -"checksum capnp-rpc 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce375f1b92bb3ae0eb3edc7409528cbfb17f35612feca67e49c38a6f29570046" "checksum capnpc 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c1a2eead5ee094e8adb30edd6ae55a139b42b8dbe27cc7e4007553ea2b5eb01" -"checksum capnpc 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "097449bb52d9c96340c88e9726be5ad8dea78bd6ee4a27c7e5c3fbe0d6f60056" "checksum cargo_metadata 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d1b4d380e1bab994591a24c2bdd1b054f64b60bef483a8c598c7c345bc3bbe" "checksum cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "39f75544d7bbaf57560d2168f28fd649ff9c76153874db88bdbdfd839b1a7e7d" "checksum ccl 4.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bb5cc85f53ef45d538aa683b6ea407bb8ea82a049c544f15c9980afb5f733e32" -"checksum ccl 4.5.3 (git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git)" = "" -"checksum ccl-crossbeam-epoch 0.7.2 (git+http://gitlab.nebulanet.cc/kabergstrom/ccl.git)" = "" "checksum ccl-crossbeam-epoch 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7fbaa5d81c52e5847e0d579410e9da9bef03da61be5f8df865521fd429df5cc1" "checksum ccl_owning_ref 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2baf176a1ac8acafcfe00675c2e4dcba8d52d4c27653be804281ad7157afee8f" "checksum ccl_stable_deref_trait 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "237406a6dcaea8c51dc75335c4dfc7a53a9e2aca7f6d6fbef00c82b0aa09277c" @@ -5185,7 +5081,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lmdb 0.8.0 (git+http://github.com/kabergstrom/lmdb-rs)" = "" "checksum lmdb-sys 0.8.0 (git+http://github.com/kabergstrom/lmdb-rs)" = "" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" -"checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff" "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" @@ -5254,11 +5149,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum parity-tokio-ipc 0.1.0 (git+https://github.com/NikVolf/parity-tokio-ipc)" = "" "checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5" "checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" -"checksum parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa7767817701cce701d5585b9c4db3cdd02086398322c1d7e8bf5094a96a2ce7" "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" "checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" -"checksum parking_lot_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cb88cb1cb3790baa6776844f968fea3be44956cf184fa1be5a03341f5491278c" "checksum parking_lot_core 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a7bbaa05312363e0480e1efee133fff1a09ef4a6406b65e226b9a793c223a32" "checksum paste 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1f4a4a1c555c6505821f9d58b8779d0f630a6b7e4e1be24ba718610acf01fa79" "checksum paste-impl 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "26e796e623b8b257215f27e6c80a5478856cae305f5b59810ff9acdaa34570e6" diff --git a/daemon/Cargo.toml b/daemon/Cargo.toml index 3b009bc3..7e7ce59a 100644 --- a/daemon/Cargo.toml +++ b/daemon/Cargo.toml @@ -21,7 +21,7 @@ mopa = "0.2.2" serde = "1.0" serde_derive = "1.0" erased-serde = "0.3" -bincode = "1.0.1" +bincode = "1.1" ron = "0.4" digest = "0.8.0" meowhash = "0.1.2" diff --git a/daemon/src/daemon.rs b/daemon/src/daemon.rs index e38fa463..fa5a7b28 100644 --- a/daemon/src/daemon.rs +++ b/daemon/src/daemon.rs @@ -12,7 +12,22 @@ use std::{ thread, }; -pub type ImporterMap = HashMap<&'static str, Box>; +#[derive(Default)] +pub struct ImporterMap(HashMap<&'static str, Box>); + +impl ImporterMap { + pub fn insert(&mut self, ext: &'static str, importer: Box) { + self.0.insert(ext, importer); + } + + pub fn get_by_path<'a>(&'a self, path: &PathBuf) -> Option<&'a dyn BoxedImporter> { + let lower_extension = path + .extension() + .map(|s| s.to_str().unwrap().to_lowercase()) + .unwrap_or_else(|| "".to_string()); + self.0.get(lower_extension.as_str()).map(|i| i.as_ref()) + } +} pub struct AssetDaemon { db_dir: PathBuf, diff --git a/daemon/src/file_asset_source.rs b/daemon/src/file_asset_source.rs index 9cba0ba0..cfdde0ad 100644 --- a/daemon/src/file_asset_source.rs +++ b/daemon/src/file_asset_source.rs @@ -1,37 +1,24 @@ use crate::asset_hub::{self, AssetHub}; -use crate::capnp_db::{DBTransaction, Environment, MessageReader, RoTransaction, RwTransaction}; +use crate::capnp_db::{CapnpCursor, DBTransaction, Environment, MessageReader, RwTransaction}; use crate::daemon::ImporterMap; -use crate::error::{Error, Result}; +use crate::error::Result; use crate::file_tracker::{FileState, FileTracker, FileTrackerEvent}; use crate::serialized_asset::SerializedAsset; -use crate::utils; -use crate::watcher::file_metadata; -use atelier_importer::{ - AssetMetadata, AssetUUID, BoxedImporter, SerdeObj, SourceMetadata as ImporterSourceMetadata, - SOURCEMETADATA_VERSION, +use crate::source_pair_import::{ + self, hash_file, HashedSourcePair, SourceMetadata, SourcePair, SourcePairImport, }; -use atelier_schema::data::{self, source_metadata, CompressionType}; +use crate::utils; +use atelier_importer::{AssetMetadata, AssetUUID, BoxedImporter}; +use atelier_schema::data::{self, source_metadata}; use bincode; use crossbeam_channel::{self as channel, Receiver}; use log::{debug, error, info}; use rayon::prelude::*; -use ron; use scoped_threadpool::Pool; use std::collections::HashMap; -use std::{ - ffi::OsStr, - fs, - hash::{Hash, Hasher}, - io::{self, BufRead, Read, Write}, - iter::FromIterator, - path::PathBuf, - str, - sync::Arc, -}; +use std::{iter::FromIterator, path::PathBuf, str, sync::Arc}; use time::PreciseTime; -pub type SourceMetadata = ImporterSourceMetadata, Box>; - pub struct FileAssetSource { hub: Arc, tracker: Arc, @@ -50,317 +37,15 @@ struct FileAssetSourceTables { asset_id_to_path: lmdb::Database, } -// Only files get Some(hash) -#[derive(Clone, Debug)] -struct HashedAssetFilePair { - source: Option, - source_hash: Option, - meta: Option, - meta_hash: Option, -} -#[derive(Clone, Debug)] -struct AssetFilePair { - source: Option, - meta: Option, -} - -struct ImportedAsset> { - asset_hash: u64, - metadata: AssetMetadata, - asset: Option>, -} -type ImportedAssetVec = ImportedAsset>; type SerializedAssetVec = SerializedAsset>; -#[derive(Debug)] -pub struct SavedImportMetadata<'a> { - importer_version: u32, - options_type: uuid::Bytes, - options: &'a [u8], - state_type: uuid::Bytes, - state: &'a [u8], - build_pipelines: HashMap, -} - -#[derive(Default)] -pub struct PairImport<'a> { - source: PathBuf, - importer: Option<&'a dyn BoxedImporter>, - source_hash: Option, - meta_hash: Option, - import_hash: Option, - source_metadata: Option, -} - -impl<'a> PairImport<'a> { - pub fn new(source: PathBuf) -> PairImport<'a> { - PairImport { - source, - ..Default::default() - } - } - pub fn with_source_hash(&mut self, source_hash: u64) { - self.source_hash = Some(source_hash); - } - pub fn with_meta_hash(&mut self, meta_hash: u64) { - self.meta_hash = Some(meta_hash); - } - pub fn hash_source(&mut self) -> Result<()> { - let (_, hash) = hash_file(&FileState { - path: self.source.clone(), - state: data::FileState::Exists, - last_modified: 0, - length: 0, - })?; - self.source_hash = - Some(hash.ok_or_else(|| Error::IO(io::Error::from(io::ErrorKind::NotFound)))?); - Ok(()) - } - /// Returns true if an appropriate importer was found, otherwise false. - pub fn with_importer_from_map(&mut self, importers: &'a ImporterMap) -> Result { - let lower_extension = self - .source - .extension() - .map(|s| s.to_str().unwrap().to_lowercase()) - .unwrap_or_else(|| "".to_string()); - self.importer = importers.get(lower_extension.as_str()).map(|i| i.as_ref()); - Ok(self.importer.is_some()) - } - pub fn needs_source_import(&mut self, scratch_buf: &mut Vec) -> Result { - if let Some(ref metadata) = self.source_metadata { - if metadata.import_hash.is_none() { - return Ok(true); - } - if self.import_hash.is_none() { - self.import_hash = Some(self.calc_import_hash( - metadata.importer_options.as_ref(), - metadata.importer_state.as_ref(), - scratch_buf, - )?); - } - Ok(self.import_hash.unwrap() != metadata.import_hash.unwrap()) - } else { - Ok(true) - } - } - fn calc_import_hash( - &self, - options: &dyn SerdeObj, - state: &dyn SerdeObj, - scratch_buf: &mut Vec, - ) -> Result { - let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); - scratch_buf.clear(); - bincode::serialize_into(&mut *scratch_buf, &options)?; - scratch_buf.hash(&mut hasher); - scratch_buf.clear(); - bincode::serialize_into(&mut *scratch_buf, &state)?; - scratch_buf.hash(&mut hasher); - self.source_hash - .expect("cannot calculate import hash without source hash") - .hash(&mut hasher); - let importer = self - .importer - .expect("cannot calculate import hash without importer"); - importer.version().hash(&mut hasher); - importer.uuid().hash(&mut hasher); - Ok(hasher.finish()) - } - - pub fn read_metadata_from_file(&mut self, scratch_buf: &mut Vec) -> Result<()> { - let importer = self - .importer - .expect("cannot read metadata without an importer"); - let meta = to_meta_path(&self.source); - let mut f = fs::File::open(meta)?; - scratch_buf.clear(); - f.read_to_end(scratch_buf)?; - self.source_metadata = Some(importer.deserialize_metadata(scratch_buf)?); - Ok(()) - } - - pub fn default_or_saved_metadata( - &mut self, - saved_metadata: Option>, - ) -> Result<()> { - let importer = self - .importer - .expect("cannot create metadata without an importer"); - let mut options = importer.default_options(); - let mut state = importer.default_state(); - let mut build_pipelines = HashMap::new(); - if let Some(saved_metadata) = saved_metadata { - if saved_metadata.options_type == options.uuid() { - options = importer.deserialize_options(saved_metadata.options)?; - } - if saved_metadata.state_type == state.uuid() { - state = importer.deserialize_state(saved_metadata.state)?; - } - build_pipelines = saved_metadata.build_pipelines; - } - self.source_metadata = Some(SourceMetadata { - version: SOURCEMETADATA_VERSION, - import_hash: None, - importer_version: importer.version(), - importer_options: options, - importer_state: state, - assets: build_pipelines - .iter() - .map(|(id, pipeline)| AssetMetadata { - id: *id, - build_pipeline: Some(*pipeline), - ..Default::default() - }) - .collect(), - }); - Ok(()) - } - - fn import_source(&mut self, scratch_buf: &mut Vec) -> Result> { - let start_time = PreciseTime::now(); - let importer = self - .importer - .expect("cannot import source without importer"); - - let metadata = std::mem::replace(&mut self.source_metadata, None) - .expect("cannot import source file without source_metadata"); - let imported = { - let mut f = fs::File::open(&self.source)?; - importer.import_boxed(&mut f, metadata.importer_options, metadata.importer_state)? - }; - let options = imported.options; - let state = imported.state; - let imported = imported.value; - let mut imported_assets = Vec::new(); - let import_hash = self.calc_import_hash(options.as_ref(), state.as_ref(), scratch_buf)?; - for mut asset in imported.assets { - asset.search_tags.push(( - "file_name".to_string(), - Some( - self.source - .file_name() - .expect("failed to get file stem") - .to_string_lossy() - .to_string(), - ), - )); - let asset_data = &asset.asset_data; - let serialized_asset = - SerializedAsset::create(asset_data.as_ref(), CompressionType::None, scratch_buf)?; - let build_pipeline = metadata - .assets - .iter() - .find(|a| a.id == asset.id) - .map(|m| m.build_pipeline) - .unwrap_or(None); - imported_assets.push({ - ImportedAsset { - asset_hash: calc_asset_hash(&asset.id, import_hash), - metadata: AssetMetadata { - id: asset.id, - search_tags: asset.search_tags, - build_deps: asset.build_deps, - load_deps: asset.load_deps, - instantiate_deps: asset.instantiate_deps, - build_pipeline, - import_asset_type: asset_data.uuid(), - }, - asset: Some(serialized_asset), - } - }); - debug!( - "Import success {} read {} bytes", - self.source.to_string_lossy(), - scratch_buf.len(), - ); - } - info!("Imported pair in {}", start_time.to(PreciseTime::now())); - self.source_metadata = Some(SourceMetadata { - version: SOURCEMETADATA_VERSION, - import_hash: Some(import_hash), - importer_version: importer.version(), - importer_options: options, - importer_state: state, - assets: imported_assets.iter().map(|m| m.metadata.clone()).collect(), - }); - Ok(imported_assets) - } - - pub fn write_metadata(&self) -> Result<()> { - let serialized_metadata = ron::ser::to_string_pretty( - self.source_metadata - .as_ref() - .expect("source_metadata missing"), - ron::ser::PrettyConfig::default(), - ) - .unwrap(); - let meta_path = to_meta_path(&self.source); - let mut meta_file = fs::File::create(meta_path)?; - meta_file.write_all(serialized_metadata.as_bytes())?; - Ok(()) - } -} - -fn to_meta_path(p: &PathBuf) -> PathBuf { - p.with_file_name(OsStr::new( - &(p.file_name().unwrap().to_str().unwrap().to_owned() + ".meta"), - )) -} - -fn calc_asset_hash(id: &AssetUUID, import_hash: u64) -> u64 { - let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); - import_hash.hash(&mut hasher); - id.hash(&mut hasher); - hasher.finish() -} - -fn hash_file(state: &FileState) -> Result<(FileState, Option)> { - let metadata = match fs::metadata(&state.path) { - Err(e) => return Err(Error::IO(e)), - Ok(m) => { - if !m.is_file() { - return Ok((state.clone(), None)); - } - file_metadata(&m) - } - }; - Ok(fs::OpenOptions::new() - .read(true) - .open(&state.path) - .and_then(|f| { - let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); - let mut reader = ::std::io::BufReader::with_capacity(64000, f); - loop { - let length = { - let buffer = reader.fill_buf()?; - hasher.write(buffer); - buffer.len() - }; - if length == 0 { - break; - } - reader.consume(length); - } - Ok(( - FileState { - path: state.path.clone(), - state: data::FileState::Exists, - last_modified: metadata.last_modified, - length: metadata.length, - }, - Some(hasher.finish()), - )) - }) - .map_err(Error::IO)?) -} - -fn hash_files<'a, T, I>(pairs: I) -> Vec> +fn hash_files<'a, T, I>(pairs: I) -> Vec> where - I: IntoParallelIterator, - T: ParallelIterator, + I: IntoParallelIterator, + T: ParallelIterator, { Vec::from_par_iter(pairs.into_par_iter().map(|s| { - let mut hashed_pair = HashedAssetFilePair { + let mut hashed_pair = HashedSourcePair { meta: s.meta.clone(), source: s.source.clone(), source_hash: None, @@ -385,25 +70,6 @@ where Ok(hashed_pair) })) } -pub fn get_saved_import_metadata<'a>( - metadata: &source_metadata::Reader<'a>, -) -> Result> { - let mut build_pipelines = HashMap::new(); - for pair in metadata.get_build_pipelines()?.iter() { - build_pipelines.insert( - utils::uuid_from_slice(&pair.get_key()?.get_id()?)?, - utils::uuid_from_slice(&pair.get_value()?.get_id()?)?, - ); - } - Ok(SavedImportMetadata { - importer_version: metadata.get_importer_version(), - options_type: utils::uuid_from_slice(metadata.get_importer_options_type()?)?, - options: metadata.get_importer_options()?, - state_type: utils::uuid_from_slice(metadata.get_importer_state_type()?)?, - state: metadata.get_importer_state()?, - build_pipelines: build_pipelines, - }) -} impl FileAssetSource { pub fn new( @@ -440,6 +106,7 @@ impl FileAssetSource { let mut value = value_builder.init_root::>(); { value.set_importer_version(metadata.importer_version); + value.set_importer_type(&metadata.importer_type); value.set_importer_state_type(&metadata.importer_state.uuid()); let mut state_buf = Vec::new(); bincode::serialize_into(&mut state_buf, &metadata.importer_state)?; @@ -490,6 +157,21 @@ impl FileAssetSource { Ok(txn.get::(self.tables.path_to_metadata, &key)?) } + pub fn iter_metadata<'a, V: DBTransaction<'a, T>, T: lmdb::Transaction + 'a>( + &self, + txn: &'a V, + ) -> Result)>>> + { + Ok(txn + .open_ro_cursor(self.tables.path_to_metadata)? + .capnp_iter_start() + .map(|(key, value)| { + let evt = value?.into_typed::(); + let path = PathBuf::from(str::from_utf8(key).expect("failed to parse key as utf8")); + Ok((path, evt)) + })) + } + fn delete_metadata(&self, txn: &mut RwTransaction<'_>, path: &PathBuf) -> Result { let key_str = path.to_string_lossy(); let key = key_str.as_bytes(); @@ -533,33 +215,22 @@ impl FileAssetSource { ) -> Result> { let path = self.get_asset_path(txn, id)?; if let Some(path) = path { - let metadata = self.get_metadata(txn, &path)?; - let saved_metadata = if let Some(ref metadata) = metadata { - Some(get_saved_import_metadata(&metadata.get()?)?) - } else { - None + let cache = DBSourceMetadataCache { + txn, + file_asset_source: self, + _marker: std::marker::PhantomData, }; - let mut import = PairImport::new(path); + let mut import = SourcePairImport::new(path); import.with_importer_from_map(&self.importers)?; - import.default_or_saved_metadata(saved_metadata)?; + import.generate_source_metadata(&cache)?; import.hash_source()?; let imported_assets = import.import_source(scratch_buf)?; Ok(imported_assets .into_iter() .find(|a| a.metadata.id == *id) .map(|a| { - a.asset.map(|a| { - ( - calc_asset_hash( - id, - import - .source_metadata - .map(|m| m.import_hash.unwrap()) - .unwrap(), - ), - a, - ) - }) + a.asset + .map(|a| (utils::calc_asset_hash(id, import.import_hash().unwrap()), a)) }) .unwrap_or(None)) } else { @@ -567,176 +238,6 @@ impl FileAssetSource { } } - fn process_pair_cases( - &self, - txn: &RoTransaction<'_>, - pair: &HashedAssetFilePair, - scratch_buf: &mut Vec, - ) -> Result, Option>)>> { - let original_pair = pair.clone(); - let mut pair = pair.clone(); - // When source or meta gets deleted, the FileState has a `state` of `Deleted`. - // For the following pattern matching, we don't want to care about the distinction between this and absence of a file. - if let HashedAssetFilePair { - source: - Some(FileState { - state: data::FileState::Deleted, - .. - }), - .. - } = pair - { - pair.source = None; - } - if let HashedAssetFilePair { - meta: - Some(FileState { - state: data::FileState::Deleted, - .. - }), - .. - } = pair - { - pair.meta = None; - } - - match pair { - // Source file has been deleted - HashedAssetFilePair { - meta: None, - source: None, - .. - } => { - if let HashedAssetFilePair { - source: Some(state), - .. - } = original_pair - { - debug!("deleted pair {}", state.path.to_string_lossy()); - } else if let HashedAssetFilePair { - meta: Some(state), .. - } = original_pair - { - debug!("deleted pair {}", state.path.to_string_lossy()); - } - Ok(None) - } - // Source file with metadata - HashedAssetFilePair { - meta: Some(_meta), - meta_hash: Some(meta_hash), - source: Some(source), - source_hash: Some(source_hash), - } => { - debug!("full pair {}", source.path.to_string_lossy()); - let mut import = PairImport::new(source.path); - import.with_source_hash(source_hash); - import.with_meta_hash(meta_hash); - if !import.with_importer_from_map(&self.importers)? { - Ok(None) - } else { - import.read_metadata_from_file(scratch_buf)?; - if import.needs_source_import(scratch_buf)? { - let imported_assets = import.import_source(scratch_buf)?; - import.write_metadata()?; - Ok(Some((import, Some(imported_assets)))) - } else { - Ok(Some((import, None))) - } - } - } - // Source file with no metadata - HashedAssetFilePair { - meta: None, - source: Some(source), - source_hash: Some(hash), - .. - } => { - debug!("file without meta {}", source.path.to_string_lossy()); - let metadata = self.get_metadata(txn, &source.path)?; - let saved_metadata = if let Some(ref m) = metadata { - Some(get_saved_import_metadata(&m.get()?)?) - } else { - None - }; - let mut import = PairImport::new(source.path); - import.with_source_hash(hash); - if !import.with_importer_from_map(&self.importers)? { - debug!("file has no importer registered"); - Ok(None) - } else { - import.default_or_saved_metadata(saved_metadata)?; - if import.needs_source_import(scratch_buf)? { - let imported_assets = import.import_source(scratch_buf)?; - import.write_metadata()?; - Ok(Some((import, Some(imported_assets)))) - } else { - Ok(Some((import, None))) - } - } - } - HashedAssetFilePair { - meta: Some(_meta), - meta_hash: Some(_hash), - source: Some(source), - source_hash: None, - } => { - debug!("directory {}", source.path.to_string_lossy()); - Ok(None) - } - HashedAssetFilePair { - meta: Some(_meta), - meta_hash: None, - source: Some(source), - source_hash: None, - } => { - debug!( - "directory with meta directory?? {}", - source.path.to_string_lossy() - ); - Ok(None) - } - HashedAssetFilePair { - meta: Some(_meta), - meta_hash: None, - source: Some(source), - source_hash: Some(_hash), - } => { - debug!( - "source file with meta directory?? {}", - source.path.to_string_lossy() - ); - Ok(None) - } - HashedAssetFilePair { - meta: None, - source: Some(source), - source_hash: None, - .. - } => { - debug!("directory with no meta {}", source.path.to_string_lossy()); - Ok(None) - } - HashedAssetFilePair { - meta: Some(meta), - meta_hash: Some(_meta_hash), - source: None, - .. - } => { - debug!( - "meta file without source file {}", - meta.path.to_string_lossy() - ); - fs::remove_file(&meta.path)?; - Ok(None) - } - _ => { - debug!("Unknown case for {:?}", pair); - Ok(None) - } - } - } - fn process_metadata_changes( &self, txn: &mut RwTransaction<'_>, @@ -804,7 +305,7 @@ impl FileAssetSource { maybe_metadata.expect("metadata exists in DB but not in hashmap"); self.hub.update_asset( txn, - calc_asset_hash( + utils::calc_asset_hash( &asset, changes .get(path) @@ -830,7 +331,7 @@ impl FileAssetSource { fn ack_dirty_file_states( &self, txn: &mut RwTransaction<'_>, - pair: &HashedAssetFilePair, + pair: &HashedSourcePair, ) -> Result<()> { let mut skip_ack_dirty = false; { @@ -863,15 +364,12 @@ impl FileAssetSource { let rename_events = self.tracker.read_rename_events(txn)?; debug!("rename events"); for (_, evt) in rename_events.iter() { - let src_str = evt.src.to_string_lossy(); - let src = src_str.as_bytes(); let dst_str = evt.dst.to_string_lossy(); let dst = dst_str.as_bytes(); let mut asset_ids = Vec::new(); let mut existing_metadata = None; { - let metadata = - txn.get::(self.tables.path_to_metadata, &src)?; + let metadata = self.get_metadata(txn, &evt.src)?; if let Some(metadata) = metadata { let metadata = metadata.get()?; let mut copy = capnp::message::Builder::new_default(); @@ -886,14 +384,8 @@ impl FileAssetSource { txn.delete(self.tables.asset_id_to_path, &asset)?; txn.put_bytes(self.tables.asset_id_to_path, &asset, &dst)?; } - debug!( - "src {} dst {} had metadata {}", - src_str, - dst_str, - existing_metadata.is_some() - ); if let Some(existing_metadata) = existing_metadata { - txn.delete(self.tables.path_to_metadata, &src)?; + self.delete_metadata(txn, &evt.src)?; txn.put(self.tables.path_to_metadata, &dst, &existing_metadata)?; } } @@ -902,6 +394,37 @@ impl FileAssetSource { } Ok(()) } + fn check_for_importer_changes(&self) -> Result { + let mut changed_paths = Vec::new(); + { + let txn = self.db.ro_txn()?; + for result in self.iter_metadata(&txn)? { + let (path, metadata) = result?; + let metadata = metadata.get()?; + let changed = if let Some(importer) = self.importers.get_by_path(&path) { + metadata.get_importer_version() != importer.version() + || metadata.get_importer_options_type()? + != importer.default_options().uuid() + || metadata.get_importer_state_type()? != importer.default_state().uuid() + || metadata.get_importer_type()? != importer.uuid() + } else { + false + }; + if changed { + changed_paths.push(path); + } + } + } + if !changed_paths.is_empty() { + let mut txn = self.db.rw_txn()?; + for path in changed_paths { + self.tracker.add_dirty_file(&mut txn, &path)?; + } + txn.commit()?; + return Ok(true); + } + Ok(false) + } fn handle_update(&self, thread_pool: &mut Pool) -> Result<()> { let start_time = PreciseTime::now(); let source_meta_pairs = { @@ -910,7 +433,7 @@ impl FileAssetSource { // This must be done in the same transaction to stay consistent. self.handle_rename_events(&mut txn)?; - let mut source_meta_pairs: HashMap = HashMap::new(); + let mut source_meta_pairs: HashMap = HashMap::new(); let dirty_files = self.tracker.read_dirty_files(&txn)?; if !dirty_files.is_empty() { for state in dirty_files.into_iter() { @@ -925,7 +448,7 @@ impl FileAssetSource { } else { state.path.clone() }; - let mut pair = source_meta_pairs.entry(base_path).or_insert(AssetFilePair { + let mut pair = source_meta_pairs.entry(base_path).or_insert(SourcePair { source: Option::None, meta: Option::None, }); @@ -937,7 +460,9 @@ impl FileAssetSource { } for (path, pair) in source_meta_pairs.iter_mut() { if pair.meta.is_none() { - pair.meta = self.tracker.get_file_state(&txn, &to_meta_path(&path))?; + pair.meta = self + .tracker + .get_file_state(&txn, &utils::to_meta_path(&path))?; } if pair.source.is_none() { pair.source = self.tracker.get_file_state(&txn, &path)?; @@ -999,8 +524,14 @@ impl FileAssetSource { sender.send((processed_pair, Err(e))).unwrap(); } Ok(read_txn) => { - let result = self.process_pair_cases( - &read_txn, + let cache = DBSourceMetadataCache { + txn: &read_txn, + file_asset_source: &self, + _marker: std::marker::PhantomData, + }; + let result = source_pair_import::process_pair( + &cache, + &self.importers, &processed_pair, local_store.as_mut().unwrap(), ); @@ -1036,7 +567,7 @@ impl FileAssetSource { // TODO put import artifact in cache metadata_changes.insert( path.clone(), - result.map(|r| r.0.source_metadata.unwrap()), + result.map(|r| r.0.source_metadata()).unwrap_or(None), ); } Err(e) => error!("Error processing pair: {}", e), @@ -1070,11 +601,24 @@ impl FileAssetSource { pub fn run(&self) -> Result<()> { let mut thread_pool = Pool::new(num_cpus::get() as u32); + let mut started = false; + let mut update = false; loop { match self.rx.recv() { - Ok(_evt) => { - self.handle_update(&mut thread_pool)?; - } + Ok(evt) => match evt { + FileTrackerEvent::Start => { + started = true; + if self.check_for_importer_changes()? || update { + self.handle_update(&mut thread_pool)?; + } + } + FileTrackerEvent::Update => { + update = true; + if started { + self.handle_update(&mut thread_pool)?; + } + } + }, Err(_) => { return Ok(()); } @@ -1082,3 +626,54 @@ impl FileAssetSource { } } } + +struct DBSourceMetadataCache<'a, 'b, V: DBTransaction<'a, T>, T: lmdb::Transaction + 'a> { + txn: &'a V, + file_asset_source: &'b FileAssetSource, + _marker: std::marker::PhantomData, +} + +impl<'a, 'b, V: DBTransaction<'a, T>, T: lmdb::Transaction + 'a> + source_pair_import::SourceMetadataCache for DBSourceMetadataCache<'a, 'b, V, T> +{ + fn restore_metadata( + &self, + path: &PathBuf, + importer: &dyn BoxedImporter, + metadata: &mut SourceMetadata, + ) -> Result<()> { + let saved_metadata = self.file_asset_source.get_metadata(self.txn, path)?; + if let Some(saved_metadata) = saved_metadata { + let saved_metadata = saved_metadata.get()?; + let mut build_pipelines = HashMap::new(); + for pair in saved_metadata.get_build_pipelines()?.iter() { + build_pipelines.insert( + utils::uuid_from_slice(&pair.get_key()?.get_id()?)?, + utils::uuid_from_slice(&pair.get_value()?.get_id()?)?, + ); + } + if saved_metadata.get_importer_options_type()? == metadata.importer_options.uuid() { + if let Ok(options) = + importer.deserialize_options(saved_metadata.get_importer_options()?) + { + metadata.importer_options = options; + } + } + if saved_metadata.get_importer_state_type()? == metadata.importer_state.uuid() { + if let Ok(state) = importer.deserialize_state(saved_metadata.get_importer_state()?) + { + metadata.importer_state = state; + } + } + metadata.assets = build_pipelines + .iter() + .map(|(id, pipeline)| AssetMetadata { + id: *id, + build_pipeline: Some(*pipeline), + ..Default::default() + }) + .collect(); + } + Ok(()) + } +} diff --git a/daemon/src/file_tracker.rs b/daemon/src/file_tracker.rs index 8dbadaef..eafa6646 100644 --- a/daemon/src/file_tracker.rs +++ b/daemon/src/file_tracker.rs @@ -1,7 +1,7 @@ use crate::capnp_db::{ CapnpCursor, DBTransaction, Environment, MessageReader, RoTransaction, RwTransaction, }; -use crate::error::Result; +use crate::error::{Error, Result}; use crate::utils; use crate::watcher::{self, FileEvent, FileMetadata}; use atelier_schema::data::{self, dirty_file_info, rename_file_event, source_file_info, FileType}; @@ -33,7 +33,8 @@ struct FileTrackerTables { } #[derive(Copy, Clone, Debug)] pub enum FileTrackerEvent { - Updated, + Start, + Update, } pub struct FileTracker { db: Arc, @@ -157,156 +158,173 @@ where Ok(()) } -fn handle_file_event( - txn: &mut RwTransaction<'_>, - tables: &FileTrackerTables, - evt: watcher::FileEvent, - scan_stack: &mut Vec, -) -> Result<()> { - match evt { - FileEvent::Updated(path, metadata) => { - let path_str = path.to_string_lossy(); - let key = path_str.as_bytes(); - let mut changed = true; - { - let maybe_msg: Option> = - txn.get(tables.source_files, &key)?; - if let Some(msg) = maybe_msg { - let info = msg.get()?; - if info.get_length() == metadata.length - && info.get_last_modified() == metadata.last_modified - && info.get_type()? == db_file_type(metadata.file_type) - { - changed = false; - } else { - debug!("CHANGED {} metadata {:?}", path_str, metadata); - } +mod events { + use super::*; + fn handle_update( + txn: &mut RwTransaction<'_>, + tables: &FileTrackerTables, + path: &PathBuf, + metadata: &watcher::FileMetadata, + scan_stack: &mut Vec, + ) -> Result<()> { + let path_str = path.to_string_lossy(); + let key = path_str.as_bytes(); + let mut changed = true; + { + let maybe_msg: Option> = + txn.get(tables.source_files, &key)?; + if let Some(msg) = maybe_msg { + let info = msg.get()?; + if info.get_length() == metadata.length + && info.get_last_modified() == metadata.last_modified + && info.get_type()? == db_file_type(metadata.file_type) + { + changed = false; + } else { + debug!("CHANGED {} metadata {:?}", path_str, metadata); } } - if !scan_stack.is_empty() { - let head_idx = scan_stack.len() - 1; - let scan_ctx = scan_stack.index_mut(head_idx); - scan_ctx.files.insert(path.clone(), metadata.clone()); - } - if changed { - let value = build_source_info(&metadata); - let dirty_value = build_dirty_file_info( - data::FileState::Exists, - value.get_root_as_reader::>()?, - ); - txn.put(tables.source_files, &key, &value)?; - txn.put(tables.dirty_files, &key, &dirty_value)?; - } } - FileEvent::Renamed(src, dst, metadata) => { - if !scan_stack.is_empty() { - let head_idx = scan_stack.len() - 1; - let scan_ctx = scan_stack.index_mut(head_idx); - scan_ctx.files.insert(dst.clone(), metadata.clone()); - scan_ctx.files.remove(&src); - } - let src_str = src.to_string_lossy(); - let src_key = src_str.as_bytes(); - let dst_str = dst.to_string_lossy(); - let dst_key = dst_str.as_bytes(); - debug!("rename {} to {} metadata {:?}", src_str, dst_str, metadata); + if !scan_stack.is_empty() { + let head_idx = scan_stack.len() - 1; + let scan_ctx = scan_stack.index_mut(head_idx); + scan_ctx.files.insert(path.clone(), metadata.clone()); + } + if changed { let value = build_source_info(&metadata); - txn.delete(tables.source_files, &src_key)?; - txn.put(tables.source_files, &dst_key, &value)?; - let dirty_value_new = build_dirty_file_info( + let dirty_value = build_dirty_file_info( data::FileState::Exists, value.get_root_as_reader::>()?, ); - let dirty_value_old = build_dirty_file_info( - data::FileState::Deleted, - value.get_root_as_reader::>()?, - ); - txn.put(tables.dirty_files, &src_key, &dirty_value_old)?; - txn.put(tables.dirty_files, &dst_key, &dirty_value_new)?; - add_rename_event(tables, txn, &src_key, &dst_key)?; + txn.put(tables.source_files, &key, &value)?; + txn.put(tables.dirty_files, &key, &dirty_value)?; } - FileEvent::Removed(path) => { - if !scan_stack.is_empty() { - let head_idx = scan_stack.len() - 1; - let scan_ctx = scan_stack.index_mut(head_idx); - scan_ctx.files.remove(&path); + Ok(()) + } + + + pub(super) fn handle_file_event( + txn: &mut RwTransaction<'_>, + tables: &FileTrackerTables, + evt: watcher::FileEvent, + scan_stack: &mut Vec, + ) -> Result> { + match evt { + FileEvent::Updated(path, metadata) => { + handle_update(txn, tables, &path, &metadata, scan_stack)?; } - let path_str = path.to_string_lossy(); - let key = path_str.as_bytes(); - debug!("removed {}", path_str); - update_deleted_dirty_entry(txn, &tables, &key)?; - txn.delete(tables.source_files, &key)?; - } - FileEvent::FileError(err) => { - debug!("file event error: {}", err); - return Err(err); - } - FileEvent::ScanStart(path) => { - debug!("scan start: {}", path.to_string_lossy()); - scan_stack.push(ScanContext { - path, - files: HashMap::new(), - }); - } - FileEvent::ScanEnd(path, watched_dirs) => { - // When we finish a scan, we know which files exist in the subdirectories. - // This means we can scan our DB for files we've tracked and delete removed files from DB - let scan_ctx = scan_stack.pop().unwrap(); - let mut db_file_set = HashSet::new(); - { + FileEvent::Renamed(src, dst, metadata) => { + if !scan_stack.is_empty() { + let head_idx = scan_stack.len() - 1; + let scan_ctx = scan_stack.index_mut(head_idx); + scan_ctx.files.insert(dst.clone(), metadata.clone()); + scan_ctx.files.remove(&src); + } + let src_str = src.to_string_lossy(); + let src_key = src_str.as_bytes(); + let dst_str = dst.to_string_lossy(); + let dst_key = dst_str.as_bytes(); + debug!("rename {} to {} metadata {:?}", src_str, dst_str, metadata); + let value = build_source_info(&metadata); + txn.delete(tables.source_files, &src_key)?; + txn.put(tables.source_files, &dst_key, &value)?; + let dirty_value_new = build_dirty_file_info( + data::FileState::Exists, + value.get_root_as_reader::>()?, + ); + let dirty_value_old = build_dirty_file_info( + data::FileState::Deleted, + value.get_root_as_reader::>()?, + ); + txn.put(tables.dirty_files, &src_key, &dirty_value_old)?; + txn.put(tables.dirty_files, &dst_key, &dirty_value_new)?; + add_rename_event(tables, txn, &src_key, &dst_key)?; + } + FileEvent::Removed(path) => { + if !scan_stack.is_empty() { + let head_idx = scan_stack.len() - 1; + let scan_ctx = scan_stack.index_mut(head_idx); + scan_ctx.files.remove(&path); + } let path_str = path.to_string_lossy(); let key = path_str.as_bytes(); - let path_string = scan_ctx.path.to_string_lossy().into_owned(); - let mut cursor = txn.open_ro_cursor(tables.source_files)?; - for (key, _) in cursor.capnp_iter_from(&key) { - let key = str::from_utf8(key).expect("Encoded key was invalid utf8"); - if !key.starts_with(&path_string) { - break; - } - db_file_set.insert(PathBuf::from(key)); - } + debug!("removed {}", path_str); + update_deleted_dirty_entry(txn, &tables, &key)?; + txn.delete(tables.source_files, &key)?; } - let scan_ctx_set = HashSet::from_iter(scan_ctx.files.keys().cloned()); - let to_remove = db_file_set.difference(&scan_ctx_set); - for p in to_remove { - let p_str = p.to_string_lossy(); - let p_key = p_str.as_bytes(); - update_deleted_dirty_entry(txn, &tables, &p_key)?; - txn.delete(tables.source_files, &p_key)?; + FileEvent::FileError(err) => { + debug!("file event error: {}", err); + return Err(err); } - info!( - "Scanned and compared {} + {}, deleted {}", - scan_ctx_set.len(), - db_file_set.len(), - db_file_set.difference(&scan_ctx_set).count() - ); - // If this is the top-level scan, we have a final set of watched directories, - // so we can delete any files that are not in any watched directories from the DB. - if scan_stack.is_empty() { - let mut to_delete = Vec::new(); + FileEvent::ScanStart(path) => { + debug!("scan start: {}", path.to_string_lossy()); + scan_stack.push(ScanContext { + path, + files: HashMap::new(), + }); + } + FileEvent::ScanEnd(path, watched_dirs) => { + // When we finish a scan, we know which files exist in the subdirectories. + // This means we can scan our DB for files we've tracked and delete removed files from DB + let scan_ctx = scan_stack.pop().unwrap(); + let mut db_file_set = HashSet::new(); { + let path_str = path.to_string_lossy(); + let key = path_str.as_bytes(); + let path_string = scan_ctx.path.to_string_lossy().into_owned(); let mut cursor = txn.open_ro_cursor(tables.source_files)?; - let dirs_as_strings = Vec::from_iter( - watched_dirs - .into_iter() - .map(|f| f.to_string_lossy().into_owned()), - ); - for (key_bytes, _) in cursor.iter_start() { - let key = str::from_utf8(key_bytes).expect("Encoded key was invalid utf8"); - if !dirs_as_strings.iter().any(|dir| key.starts_with(dir)) { - to_delete.push(key); + for (key, _) in cursor.capnp_iter_from(&key) { + let key = str::from_utf8(key).expect("Encoded key was invalid utf8"); + if !key.starts_with(&path_string) { + break; } + db_file_set.insert(PathBuf::from(key)); } } - for key in to_delete { - txn.delete(tables.source_files, &key)?; - update_deleted_dirty_entry(txn, &tables, &key)?; + let scan_ctx_set = HashSet::from_iter(scan_ctx.files.keys().cloned()); + let to_remove = db_file_set.difference(&scan_ctx_set); + for p in to_remove { + let p_str = p.to_string_lossy(); + let p_key = p_str.as_bytes(); + update_deleted_dirty_entry(txn, &tables, &p_key)?; + txn.delete(tables.source_files, &p_key)?; + } + info!( + "Scanned and compared {} + {}, deleted {}", + scan_ctx_set.len(), + db_file_set.len(), + db_file_set.difference(&scan_ctx_set).count() + ); + // If this is the top-level scan, we have a final set of watched directories, + // so we can delete any files that are not in any watched directories from the DB. + if scan_stack.is_empty() { + let mut to_delete = Vec::new(); + { + let mut cursor = txn.open_ro_cursor(tables.source_files)?; + let dirs_as_strings = Vec::from_iter( + watched_dirs + .into_iter() + .map(|f| f.to_string_lossy().into_owned()), + ); + for (key_bytes, _) in cursor.iter_start() { + let key = + str::from_utf8(key_bytes).expect("Encoded key was invalid utf8"); + if !dirs_as_strings.iter().any(|dir| key.starts_with(dir)) { + to_delete.push(key); + } + } + } + for key in to_delete { + txn.delete(tables.source_files, &key)?; + update_deleted_dirty_entry(txn, &tables, &key)?; + } } + debug!("scan end: {}", path.to_string_lossy()); + return Ok(Some(FileTrackerEvent::Start)); } - debug!("scan end: {}", path.to_string_lossy()); } + Ok(None) } - Ok(()) } impl FileTracker { @@ -328,7 +346,7 @@ impl FileTracker { rename_file_events: db .create_db(Some("rename_file_events"), lmdb::DatabaseFlags::INTEGER_KEY)?, }, - db: db, + db, listener_rx: rx, listener_tx: tx, is_running: AtomicBool::new(false), @@ -336,7 +354,7 @@ impl FileTracker { }) } - pub fn get_watch_dirs<'a>(&'a self) -> impl Iterator { + pub fn get_watch_dirs(&self) -> impl Iterator { self.watch_dirs.iter() } @@ -381,6 +399,28 @@ impl FileTracker { Ok(()) } + pub fn add_dirty_file(&self, txn: &mut RwTransaction<'_>, path: &PathBuf) -> Result<()> { + let metadata = match fs::metadata(path) { + Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(Error::IO(e)), + Ok(metadata) => Some(watcher::file_metadata(&metadata)), + }; + let path_str = path.to_string_lossy(); + let key = path_str.as_bytes(); + if let Some(metadata) = metadata { + let source_info = build_source_info(&metadata); + let dirty_file_info = build_dirty_file_info( + data::FileState::Exists, + source_info.get_root_as_reader::>()?, + ); + txn.put(self.tables.source_files, &key, &source_info)?; + txn.put(self.tables.dirty_files, &key, &dirty_file_info)?; + } else { + update_deleted_dirty_entry(txn, &self.tables, &key)?; + } + Ok(()) + } + pub fn read_dirty_files<'a, V: DBTransaction<'a, T>, T: lmdb::Transaction + 'a>( &self, iter_txn: &'a V, @@ -492,8 +532,9 @@ impl FileTracker { is_running: &mut bool, rx: &Receiver, scan_stack: &mut Vec, - ) -> Result> { + ) -> Result> { let mut txn = None; + let mut output_evts = Vec::new(); while *is_running { let timeout = Duration::from_millis(100); select! { @@ -503,7 +544,9 @@ impl FileTracker { txn = Some(self.db.rw_txn()?); } let txn = txn.as_mut().unwrap(); - handle_file_event(txn, &self.tables, evt, scan_stack)?; + if let Some(evt) = events::handle_file_event(txn, &self.tables, evt, scan_stack)? { + output_evts.push(evt); + } } else { error!("Receive error"); *is_running = false; @@ -519,13 +562,10 @@ impl FileTracker { if txn.dirty { txn.commit()?; debug!("Commit"); - Ok(Some(FileTrackerEvent::Updated)) - } else { - Ok(None) + output_evts.push(FileTrackerEvent::Update); } - } else { - Ok(None) } + Ok(output_evts) } pub fn run(&self) -> Result<()> { @@ -536,13 +576,10 @@ impl FileTracker { return Ok(()); } let (tx, rx) = channel::unbounded(); - let dir_strings: Vec<_> = self - .watch_dirs - .iter() - .map(|p| p.to_string_lossy().to_string()) - .collect(); - let mut watcher = - watcher::DirWatcher::from_path_iter(dir_strings.iter().map(|s| s.as_str()), tx)?; + let mut watcher = watcher::DirWatcher::from_path_iter( + self.watch_dirs.iter().map(|p| p.to_str().unwrap()), + tx, + )?; let stop_handle = watcher.stop_handle(); let handle = thread::spawn(move || watcher.run()); @@ -550,7 +587,7 @@ impl FileTracker { while self.is_running.load(Ordering::Acquire) { let mut scan_stack = Vec::new(); let mut is_running = true; - let event = self.read_file_events(&mut is_running, &rx, &mut scan_stack)?; + let events = self.read_file_events(&mut is_running, &rx, &mut scan_stack)?; select! { recv(self.listener_rx) -> listener => { if let Ok(listener) = listener { @@ -559,7 +596,7 @@ impl FileTracker { }, default => {} } - if let Some(event) = event { + for event in events { for listener in listeners.iter() { debug!("Sent to listener"); select! { diff --git a/daemon/src/lib.rs b/daemon/src/lib.rs index f23754b2..ae4f043a 100644 --- a/daemon/src/lib.rs +++ b/daemon/src/lib.rs @@ -9,6 +9,7 @@ pub mod error; mod file_asset_source; pub mod file_tracker; mod serialized_asset; +mod source_pair_import; mod utils; pub mod watcher; diff --git a/daemon/src/source_pair_import.rs b/daemon/src/source_pair_import.rs new file mode 100644 index 00000000..3daca2bb --- /dev/null +++ b/daemon/src/source_pair_import.rs @@ -0,0 +1,473 @@ +use crate::daemon::ImporterMap; +use crate::error::{Error, Result}; +use crate::file_tracker::FileState; +use crate::serialized_asset::SerializedAsset; +use crate::utils; +use crate::watcher::file_metadata; +use atelier_importer::{ + AssetMetadata, BoxedImporter, SerdeObj, SourceMetadata as ImporterSourceMetadata, + SOURCEMETADATA_VERSION, +}; +use atelier_schema::data::{self, CompressionType}; +use bincode; +use log::{debug, info}; +use ron; +use std::{ + fs, + hash::{Hash, Hasher}, + io::{self, BufRead, Read, Write}, + path::PathBuf, +}; +use time::PreciseTime; + +pub type SourceMetadata = ImporterSourceMetadata, Box>; + +// Only files get Some(hash) +#[derive(Clone, Debug)] +pub struct HashedSourcePair { + pub source: Option, + pub source_hash: Option, + pub meta: Option, + pub meta_hash: Option, +} +#[derive(Clone, Debug)] +pub struct SourcePair { + pub source: Option, + pub meta: Option, +} + +pub(crate) type ImportedAssetVec = ImportedAsset>; + +pub struct ImportedAsset> { + pub asset_hash: u64, + pub metadata: AssetMetadata, + pub asset: Option>, +} + +#[derive(Default)] +pub struct SourcePairImport<'a> { + source: PathBuf, + importer: Option<&'a dyn BoxedImporter>, + source_hash: Option, + meta_hash: Option, + import_hash: Option, + source_metadata: Option, +} + +pub trait SourceMetadataCache { + fn restore_metadata<'a>( + &self, + path: &PathBuf, + importer: &'a dyn BoxedImporter, + metadata: &mut SourceMetadata, + ) -> Result<()>; +} + +impl<'a> SourcePairImport<'a> { + pub fn new(source: PathBuf) -> SourcePairImport<'a> { + SourcePairImport { + source, + ..Default::default() + } + } + pub fn source_metadata(self) -> Option { + self.source_metadata + } + pub fn with_source_hash(&mut self, source_hash: u64) { + self.source_hash = Some(source_hash); + } + pub fn with_meta_hash(&mut self, meta_hash: u64) { + self.meta_hash = Some(meta_hash); + } + pub fn hash_source(&mut self) -> Result<()> { + let (_, hash) = hash_file(&FileState { + path: self.source.clone(), + state: data::FileState::Exists, + last_modified: 0, + length: 0, + })?; + self.source_hash = + Some(hash.ok_or_else(|| Error::IO(io::Error::from(io::ErrorKind::NotFound)))?); + Ok(()) + } + /// Returns true if an appropriate importer was found, otherwise false. + pub fn with_importer_from_map(&mut self, importers: &'a ImporterMap) -> Result { + self.importer = importers.get_by_path(&self.source); + Ok(self.importer.is_some()) + } + pub fn needs_source_import(&mut self, scratch_buf: &mut Vec) -> Result { + if let Some(ref metadata) = self.source_metadata { + if metadata.import_hash.is_none() { + return Ok(true); + } + if self.import_hash.is_none() { + self.import_hash = Some(self.calc_import_hash( + metadata.importer_options.as_ref(), + metadata.importer_state.as_ref(), + metadata.importer_version, + metadata.importer_type, + scratch_buf, + )?); + } + Ok(self.import_hash.unwrap() != metadata.import_hash.unwrap()) + } else { + Ok(true) + } + } + fn calc_import_hash( + &self, + options: &dyn SerdeObj, + state: &dyn SerdeObj, + importer_version: u32, + importer_type: uuid::Bytes, + scratch_buf: &mut Vec, + ) -> Result { + let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); + scratch_buf.clear(); + bincode::serialize_into(&mut *scratch_buf, &options)?; + scratch_buf.hash(&mut hasher); + scratch_buf.clear(); + bincode::serialize_into(&mut *scratch_buf, &state)?; + scratch_buf.hash(&mut hasher); + self.source_hash + .expect("cannot calculate import hash without source hash") + .hash(&mut hasher); + importer_version.hash(&mut hasher); + importer_type.hash(&mut hasher); + Ok(hasher.finish()) + } + + pub fn import_hash(&self) -> Option { + self.import_hash + } + + pub fn read_metadata_from_file(&mut self, scratch_buf: &mut Vec) -> Result<()> { + let importer = self + .importer + .expect("cannot read metadata without an importer"); + let meta = utils::to_meta_path(&self.source); + let mut f = fs::File::open(meta)?; + scratch_buf.clear(); + f.read_to_end(scratch_buf)?; + self.source_metadata = Some(importer.deserialize_metadata(scratch_buf)?); + Ok(()) + } + + pub fn generate_source_metadata( + &mut self, + metadata_cache: &C, + ) -> Result<()> { + let importer = self + .importer + .expect("cannot create metadata without an importer"); + let mut default_metadata = SourceMetadata { + version: SOURCEMETADATA_VERSION, + importer_version: importer.version(), + importer_type: importer.uuid(), + importer_options: importer.default_options(), + importer_state: importer.default_state(), + import_hash: None, + assets: Vec::new(), + }; + metadata_cache.restore_metadata(&self.source, importer, &mut default_metadata)?; + self.source_metadata = Some(default_metadata); + Ok(()) + } + + pub fn import_source(&mut self, scratch_buf: &mut Vec) -> Result> { + let start_time = PreciseTime::now(); + let importer = self + .importer + .expect("cannot import source without importer"); + + let metadata = std::mem::replace(&mut self.source_metadata, None) + .expect("cannot import source file without source_metadata"); + let imported = { + let mut f = fs::File::open(&self.source)?; + importer.import_boxed(&mut f, metadata.importer_options, metadata.importer_state)? + }; + let options = imported.options; + let state = imported.state; + let imported = imported.value; + let mut imported_assets = Vec::new(); + let import_hash = self.calc_import_hash( + options.as_ref(), + state.as_ref(), + importer.version(), + importer.uuid(), + scratch_buf, + )?; + for mut asset in imported.assets { + asset.search_tags.push(( + "file_name".to_string(), + Some( + self.source + .file_name() + .expect("failed to get file stem") + .to_string_lossy() + .to_string(), + ), + )); + let asset_data = &asset.asset_data; + let serialized_asset = + SerializedAsset::create(asset_data.as_ref(), CompressionType::None, scratch_buf)?; + let build_pipeline = metadata + .assets + .iter() + .find(|a| a.id == asset.id) + .map(|m| m.build_pipeline) + .unwrap_or(None); + imported_assets.push({ + ImportedAsset { + asset_hash: utils::calc_asset_hash(&asset.id, import_hash), + metadata: AssetMetadata { + id: asset.id, + search_tags: asset.search_tags, + build_deps: asset.build_deps, + load_deps: asset.load_deps, + instantiate_deps: asset.instantiate_deps, + build_pipeline, + import_asset_type: asset_data.uuid(), + }, + asset: Some(serialized_asset), + } + }); + debug!( + "Import success {} read {} bytes", + self.source.to_string_lossy(), + scratch_buf.len(), + ); + } + info!("Imported pair in {}", start_time.to(PreciseTime::now())); + self.source_metadata = Some(SourceMetadata { + version: SOURCEMETADATA_VERSION, + import_hash: Some(import_hash), + importer_version: importer.version(), + importer_type: importer.uuid(), + importer_options: options, + importer_state: state, + assets: imported_assets.iter().map(|m| m.metadata.clone()).collect(), + }); + Ok(imported_assets) + } + + pub fn write_metadata(&self) -> Result<()> { + let serialized_metadata = ron::ser::to_string_pretty( + self.source_metadata + .as_ref() + .expect("source_metadata missing"), + ron::ser::PrettyConfig::default(), + ) + .unwrap(); + let meta_path = utils::to_meta_path(&self.source); + let mut meta_file = fs::File::create(meta_path)?; + meta_file.write_all(serialized_metadata.as_bytes())?; + Ok(()) + } +} + +pub fn process_pair<'a, C: SourceMetadataCache>( + metadata_cache: &C, + importer_map: &'a ImporterMap, + pair: &HashedSourcePair, + scratch_buf: &mut Vec, +) -> Result, Option>)>> { + let original_pair = pair.clone(); + let mut pair = pair.clone(); + // When source or meta gets deleted, the FileState has a `state` of `Deleted`. + // For the following pattern matching, we don't want to care about the distinction between this and absence of a file. + if let HashedSourcePair { + source: + Some(FileState { + state: data::FileState::Deleted, + .. + }), + .. + } = pair + { + pair.source = None; + } + if let HashedSourcePair { + meta: + Some(FileState { + state: data::FileState::Deleted, + .. + }), + .. + } = pair + { + pair.meta = None; + } + + match pair { + // Source file has been deleted + HashedSourcePair { + meta: None, + source: None, + .. + } => { + if let HashedSourcePair { + source: Some(state), + .. + } = original_pair + { + debug!("deleted pair {}", state.path.to_string_lossy()); + } else if let HashedSourcePair { + meta: Some(state), .. + } = original_pair + { + debug!("deleted pair {}", state.path.to_string_lossy()); + } + Ok(None) + } + // Source file with metadata + HashedSourcePair { + meta: Some(_meta), + meta_hash: Some(meta_hash), + source: Some(source), + source_hash: Some(source_hash), + } => { + debug!("full pair {}", source.path.to_string_lossy()); + let mut import = SourcePairImport::new(source.path); + import.with_source_hash(source_hash); + import.with_meta_hash(meta_hash); + if !import.with_importer_from_map(&importer_map)? { + Ok(None) + } else { + import.read_metadata_from_file(scratch_buf)?; + if import.needs_source_import(scratch_buf)? { + debug!("needs source import {:?}", import.source); + let imported_assets = import.import_source(scratch_buf)?; + import.write_metadata()?; + Ok(Some((import, Some(imported_assets)))) + } else { + debug!("does not need source import {:?}", import.source); + Ok(Some((import, None))) + } + } + } + // Source file with no metadata + HashedSourcePair { + meta: None, + source: Some(source), + source_hash: Some(hash), + .. + } => { + debug!("file without meta {}", source.path.to_string_lossy()); + let mut import = SourcePairImport::new(source.path); + import.with_source_hash(hash); + if !import.with_importer_from_map(&importer_map)? { + debug!("file has no importer registered"); + Ok(Some((import, None))) + } else { + import.generate_source_metadata(metadata_cache)?; + if import.needs_source_import(scratch_buf)? { + let imported_assets = import.import_source(scratch_buf)?; + import.write_metadata()?; + Ok(Some((import, Some(imported_assets)))) + } else { + Ok(Some((import, None))) + } + } + } + HashedSourcePair { + meta: Some(_meta), + meta_hash: Some(_hash), + source: Some(source), + source_hash: None, + } => { + debug!("directory {}", source.path.to_string_lossy()); + Ok(None) + } + HashedSourcePair { + meta: Some(_meta), + meta_hash: None, + source: Some(source), + source_hash: None, + } => { + debug!( + "directory with meta directory?? {}", + source.path.to_string_lossy() + ); + Ok(None) + } + HashedSourcePair { + meta: Some(_meta), + meta_hash: None, + source: Some(source), + source_hash: Some(_hash), + } => { + debug!( + "source file with meta directory?? {}", + source.path.to_string_lossy() + ); + Ok(None) + } + HashedSourcePair { + meta: None, + source: Some(source), + source_hash: None, + .. + } => { + debug!("directory with no meta {}", source.path.to_string_lossy()); + Ok(None) + } + HashedSourcePair { + meta: Some(meta), + meta_hash: Some(_meta_hash), + source: None, + .. + } => { + debug!( + "meta file without source file {}", + meta.path.to_string_lossy() + ); + fs::remove_file(&meta.path)?; + Ok(None) + } + _ => { + debug!("Unknown case for {:?}", pair); + Ok(None) + } + } +} + +pub(crate) fn hash_file(state: &FileState) -> Result<(FileState, Option)> { + let metadata = match fs::metadata(&state.path) { + Err(e) => return Err(Error::IO(e)), + Ok(m) => { + if !m.is_file() { + return Ok((state.clone(), None)); + } + file_metadata(&m) + } + }; + Ok(fs::OpenOptions::new() + .read(true) + .open(&state.path) + .and_then(|f| { + let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); + let mut reader = ::std::io::BufReader::with_capacity(64000, f); + loop { + let length = { + let buffer = reader.fill_buf()?; + hasher.write(buffer); + buffer.len() + }; + if length == 0 { + break; + } + reader.consume(length); + } + Ok(( + FileState { + path: state.path.clone(), + state: data::FileState::Exists, + last_modified: metadata.last_modified, + length: metadata.length, + }, + Some(hasher.finish()), + )) + }) + .map_err(Error::IO)?) +} diff --git a/daemon/src/utils.rs b/daemon/src/utils.rs index dff15e2c..68bc2d36 100644 --- a/daemon/src/utils.rs +++ b/daemon/src/utils.rs @@ -1,6 +1,12 @@ use crate::error::Error; +use atelier_importer::AssetUUID; +use std::{ + ffi::OsStr, + hash::{Hash, Hasher}, + path::PathBuf, +}; -pub fn make_array(slice: &[T]) -> A +pub(crate) fn make_array(slice: &[T]) -> A where A: Sized + Default + AsMut<[T]>, T: Copy, @@ -10,7 +16,7 @@ where a } -pub fn uuid_from_slice(slice: &[u8]) -> Result { +pub(crate) fn uuid_from_slice(slice: &[u8]) -> Result { const BYTES_LEN: usize = 16; let len = slice.len(); @@ -23,3 +29,16 @@ pub fn uuid_from_slice(slice: &[u8]) -> Result { bytes.copy_from_slice(slice); Ok(bytes) } + +pub(crate) fn to_meta_path(p: &PathBuf) -> PathBuf { + p.with_file_name(OsStr::new( + &(p.file_name().unwrap().to_str().unwrap().to_owned() + ".meta"), + )) +} + +pub(crate) fn calc_asset_hash(id: &AssetUUID, import_hash: u64) -> u64 { + let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); + import_hash.hash(&mut hasher); + id.hash(&mut hasher); + hasher.finish() +} diff --git a/importer/Cargo.toml b/importer/Cargo.toml index 07d5f621..cb9aa550 100644 --- a/importer/Cargo.toml +++ b/importer/Cargo.toml @@ -10,5 +10,5 @@ serde = "1.0" type-uuid = "0.1" erased-serde = "0.3" ron = "0.4" -bincode = "1.0" +bincode = "1.1" inventory = "0.1" diff --git a/importer/src/boxed_importer.rs b/importer/src/boxed_importer.rs index 7a11d20f..37ab825d 100644 --- a/importer/src/boxed_importer.rs +++ b/importer/src/boxed_importer.rs @@ -23,6 +23,8 @@ pub struct SourceMetadata { /// Hash of the source file + importer options + importer state this metadata was generated from pub import_hash: Option, pub importer_version: u32, + #[serde(default)] + pub importer_type: AssetTypeId, pub importer_options: Options, pub importer_state: State, pub assets: Vec, @@ -90,6 +92,7 @@ where version: metadata.version, import_hash: metadata.import_hash, importer_version: metadata.importer_version, + importer_type: metadata.importer_type, importer_options: Box::new(metadata.importer_options), importer_state: Box::new(metadata.importer_state), assets: metadata.assets.clone(), @@ -110,11 +113,9 @@ pub struct SourceFileImporter { } inventory::collect!(SourceFileImporter); -pub fn get_source_importers() -> impl Iterator)> { +pub fn get_source_importers( +) -> impl Iterator)> { inventory::iter:: .into_iter() - .map(|s| ( - s.extension.trim_start_matches("."), - (s.instantiator)(), - )) + .map(|s| (s.extension.trim_start_matches("."), (s.instantiator)())) } diff --git a/loader/src/rpc_loader.rs b/loader/src/rpc_loader.rs index e7be101f..19333312 100644 --- a/loader/src/rpc_loader.rs +++ b/loader/src/rpc_loader.rs @@ -24,17 +24,28 @@ use std::{ }; use tokio::prelude::*; +/// Describes the state of an asset load operation #[derive(Copy, Clone, PartialEq, Debug)] enum LoadState { + /// Indeterminate state - may transition into a lod, or result in removal if ref count is < 0 None, + /// The load operation needs metadata to progress WaitingForMetadata, + /// Metadata is being fetched for the load operation RequestingMetadata, + /// The load operation needs asset data to progress WaitingForData, + /// Asset data is being fetched for the load operation RequestingData, - LoadingData, + /// Asset data is being decompressed + DecompressingData, + /// Asset data is being loaded by engine systems LoadingAsset, + /// Asset is loaded and ready to use Loaded, + /// Asset should be unloaded UnloadRequested, + /// Asset is being unloaded by engine systems Unloading, } @@ -147,7 +158,7 @@ impl Loader for RpcLoader { .map(|s| match s.state { None => LoadStatus::NotRequested, WaitingForMetadata | RequestingMetadata | WaitingForData | RequestingData - | LoadingData | LoadingAsset => LoadStatus::Loading, + | DecompressingData | LoadingAsset => LoadStatus::Loading, Loaded => { if let Some(_) = s.loaded_version { LoadStatus::Loaded @@ -538,7 +549,7 @@ fn process_load_states( } } LoadState::RequestingData => LoadState::RequestingData, - LoadState::LoadingData => LoadState::LoadingData, + LoadState::DecompressingData => LoadState::DecompressingData, LoadState::LoadingAsset => LoadState::LoadingAsset, LoadState::Loaded => { if value.refs.load(Ordering::Relaxed) <= 0 { diff --git a/schema/schemas/data.capnp b/schema/schemas/data.capnp index 1238c13a..10722c8a 100644 --- a/schema/schemas/data.capnp +++ b/schema/schemas/data.capnp @@ -58,6 +58,7 @@ struct SourceMetadata { importerStateType @4 :Data; importerState @5 :Data; buildPipelines @6 :List(AssetUuidPair); + importerType @7 :Data; } struct ImportError {