diff --git a/Cargo.lock b/Cargo.lock index deb7fc27f7..d36a993abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8486,6 +8486,7 @@ dependencies = [ "polymesh-dart", "polymesh-primitives", "polymesh-runtime-common", + "polymesh-worker-common", "polymesh-worker-extension", "polymesh-worker-protocol-dart-v1", "rand_chacha 0.3.1", @@ -9403,6 +9404,45 @@ dependencies = [ "substrate-test-utils", ] +[[package]] +name = "pallet-worker-modules" +version = "8.1.0" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "polymesh-primitives", + "polymesh-worker-common", + "polymesh-worker-extension", + "scale-info", + "serde", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-worker-testing" +version = "0.1.0" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "polymesh-primitives", + "polymesh-runtime-common", + "polymesh-worker-common", + "polymesh-worker-extension", + "polymesh-worker-protocol-testing", + "scale-info", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "parity-db" version = "0.4.13" @@ -10796,6 +10836,8 @@ dependencies = [ "pallet-treasury", "pallet-utility", "pallet-validators", + "pallet-worker-modules", + "pallet-worker-testing", "parity-scale-codec", "polymesh-build-tool", "polymesh-dart", @@ -10883,6 +10925,7 @@ dependencies = [ "pallet-treasury", "pallet-utility", "pallet-validators", + "pallet-worker-modules", "parity-scale-codec", "polymesh-build-tool", "polymesh-primitives", @@ -10972,6 +11015,7 @@ dependencies = [ "pallet-treasury", "pallet-utility", "pallet-validators", + "pallet-worker-modules", "parity-scale-codec", "polymesh-build-tool", "polymesh-dart", @@ -11064,6 +11108,7 @@ dependencies = [ "pallet-treasury", "pallet-utility", "pallet-validators", + "pallet-worker-modules", "parity-scale-codec", "polymesh-exec-macro", "polymesh-precompiles", @@ -11198,6 +11243,8 @@ dependencies = [ "log", "parity-scale-codec", "polkavm-derive 0.32.0", + "scale-info", + "serde", ] [[package]] @@ -11222,9 +11269,11 @@ name = "polymesh-worker-native" version = "0.1.0" dependencies = [ "log", + "parity-scale-codec", "polymesh-worker", "polymesh-worker-common", "polymesh-worker-protocol-dart-v1", + "polymesh-worker-protocol-testing", "sp-panic-handler", ] @@ -11249,6 +11298,21 @@ dependencies = [ "sp-std", ] +[[package]] +name = "polymesh-worker-protocol-testing" +version = "0.1.0" +dependencies = [ + "bounded-collections", + "log", + "parity-scale-codec", + "picoalloc", + "polkavm-derive 0.33.0", + "polymesh-worker-common", + "polymesh-worker-extension", + "scale-info", + "sp-std", +] + [[package]] name = "polymesh-worker-tester" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0a4086094a..e2aa6b8ce8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ members = [ "worker/common", "worker/native", "worker/protocol/dart-v1", + "worker/protocol/testing", "pallets/asset", "pallets/base", "pallets/committee", @@ -117,6 +118,8 @@ members = [ "pallets/treasury", "pallets/utility", "pallets/weights", + "pallets/worker-modules", + "pallets/worker-modules/worker-testing", "primitives", "primitives_derive", "primitives/asset-metadata", @@ -142,6 +145,7 @@ polymesh-worker-extension = { path = "worker/extension", default-features = fals polymesh-worker-common = { path = "worker/common", default-features = false } polymesh-worker-native = { path = "worker/native", default-features = false } polymesh-worker-protocol-dart-v1 = { path = "worker/protocol/dart-v1", default-features = false } +polymesh-worker-protocol-testing = { path = "worker/protocol/testing", default-features = false } schnellru = { version = "0.2.4", default-features = false } rayon = { version = "1.8" } @@ -177,6 +181,8 @@ polymesh-transaction-payment = { path = "pallets/transaction-payment", default-f pallet-treasury = { path = "pallets/treasury", default-features = false } pallet-utility = { path = "pallets/utility", default-features = false } pallet-precompiles = { path = "pallets/precompiles", default-features = false } +pallet-worker-modules = { path = "pallets/worker-modules", default-features = false } +pallet-worker-testing = { path = "pallets/worker-modules/worker-testing", default-features = false } # Common polymesh-runtime-common = { path = "pallets/runtime/common", default-features = false } @@ -483,6 +489,25 @@ default = [ "worker_polkavm", "worker_wasmtime", "worker_wasmer", + "host_workers", + #"no_host_fns", + #"host_curves", +] + +# Use host: worker modules + msm + Hash to Curve +host_workers = [ + "polymesh-runtime-develop/host_workers", + "polymesh-runtime-testnet/host_workers", +] +# No host functions +no_host_fns = [ + "polymesh-runtime-develop/no_host_fns", + "polymesh-runtime-testnet/no_host_fns", +] +# Verify proofs in runtime with only these host functions: msm + Hash to Curve +host_curves = [ + "polymesh-runtime-develop/host_curves", + "polymesh-runtime-testnet/host_curves", ] disable_fees = [ diff --git a/integration/src/lib.rs b/integration/src/lib.rs index 3c7c17bc09..4a6bd3d9ad 100644 --- a/integration/src/lib.rs +++ b/integration/src/lib.rs @@ -20,6 +20,9 @@ pub use asset_helper::*; #[cfg(feature = "current_release")] pub mod confidential_assets_helper; +#[cfg(feature = "current_release")] +pub mod worker_modules_helper; + #[cfg(feature = "current_release")] pub mod contracts; diff --git a/integration/src/worker_modules_helper.rs b/integration/src/worker_modules_helper.rs new file mode 100644 index 0000000000..aeb2d4f4d9 --- /dev/null +++ b/integration/src/worker_modules_helper.rs @@ -0,0 +1,138 @@ +use anyhow::Result; + +use polymesh_api::{types::polymesh_worker_common::BackendModuleDefinition, Api}; +use polymesh_api_tester::{DbAccountSigner, PolymeshTester}; + +pub use polymesh_api::types::{ + pallet_worker_modules::*, + polymesh_worker_common::{BackendModuleKind, Protocol, ProtocolId, ProtocolVersion}, + runtime::RuntimeCall, +}; +use sp_core::blake2_256; + +pub struct WorkerModulesHelper { + pub api: Api, + pub protocol: Protocol, + sudo: DbAccountSigner, +} + +impl WorkerModulesHelper { + pub fn new(tester: &PolymeshTester, protocol: Protocol) -> Self { + let api = tester.api.clone(); + let sudo = tester + .sudo + .as_ref() + .expect("Sudo account not found") + .clone(); + Self { + api, + protocol, + sudo, + } + } + + pub fn update_version(&mut self, version: ProtocolVersion) { + self.protocol.version = version; + } + + pub async fn sudo_call>(&mut self, call: C) -> Result<()> { + let mut res = self + .api + .call() + .sudo() + .sudo(call.into())? + .submit_and_watch(&mut self.sudo) + .await?; + res.wait_finalized().await?; + Ok(()) + } + + pub async fn register_protocol( + &mut self, + name: &str, + description: &str, + version: ProtocolVersion, + ) -> Result<()> { + self.sudo_call(self.api.call().worker_modules().register_protocol( + self.protocol.id.clone(), + ProtocolMetadata { + protocol_name: name.as_bytes().to_vec(), + protocol_description: description.as_bytes().to_vec(), + protocol_version: version, + }, + )?) + .await?; + + Ok(()) + } + + pub async fn upload_module_code( + &mut self, + module_code: Vec, + module_version: u32, + ) -> Result<()> { + self.sudo_call( + self.api + .call() + .worker_modules() + .upload_protocol_module_code(self.protocol.clone(), module_code, module_version)?, + ) + .await?; + + Ok(()) + } + + pub async fn upload_module_context(&mut self, module_context: Vec) -> Result<()> { + self.sudo_call( + self.api + .call() + .worker_modules() + .upload_protocol_module_context(self.protocol.clone(), module_context)?, + ) + .await?; + + Ok(()) + } + + pub async fn upload_config( + &mut self, + init_method: ProtocolInitializationMethod, + modules: Vec, + ) -> Result<()> { + let config = ProtocolModuleConfig { + protocol: self.protocol.clone(), + initialization_method: init_method.clone(), + modules: modules.clone(), + }; + self.sudo_call( + self.api + .call() + .worker_modules() + .upload_protocol_module_config(config)?, + ) + .await?; + + Ok(()) + } + + pub async fn upload_modules_and_config( + &mut self, + init_method: ProtocolInitializationMethod, + modules: Vec<(BackendModuleKind, u32, Vec)>, + ) -> Result<()> { + let mut module_defs = Vec::new(); + for (module_kind, module_version, code) in modules.into_iter() { + let code_hash = blake2_256(&code); + self.upload_module_code(code, module_version).await?; + module_defs.push(BackendModuleDefinition { + module_kind, + module_version, + code_hash, + }); + } + + self.upload_config(init_method, module_defs).await?; + + Ok(()) + } +} diff --git a/integration/tests/worker_testing.rs b/integration/tests/worker_testing.rs new file mode 100644 index 0000000000..691b7444bf --- /dev/null +++ b/integration/tests/worker_testing.rs @@ -0,0 +1,222 @@ +// >=v7.3 +#[cfg(feature = "current_release")] +mod worker_modules_tests { + use anyhow::Result; + + use integration::{worker_modules_helper::*, PolymeshTester}; + + async fn upload_v1(helper: &mut WorkerModulesHelper) -> Result<()> { + let polkavm_zst = include_bytes!( + "../../worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm.zst" + ) + .to_vec(); + let polkavm = include_bytes!( + "../../worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm" + ) + .to_vec(); + let wasm_zst = include_bytes!( + "../../worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm.zst" + ) + .to_vec(); + let wasm = include_bytes!( + "../../worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm" + ) + .to_vec(); + + // Upload the module code and config. + helper.update_version(ProtocolVersion { + major: 1, + minor: 0, + patch: 0, + }); + helper + .upload_modules_and_config( + ProtocolInitializationMethod::SaveContextFromFirstInstance, + vec![ + (BackendModuleKind::PolkaVM, 2, polkavm_zst.clone()), + (BackendModuleKind::PolkaVM, 1, polkavm.clone()), + (BackendModuleKind::Wasm, 2, wasm_zst.clone()), + (BackendModuleKind::Wasm, 1, wasm.clone()), + ], + ) + .await?; + Ok(()) + } + + async fn upload_v2(helper: &mut WorkerModulesHelper) -> Result<()> { + let polkavm_zst = include_bytes!( + "../../worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm.zst" + ) + .to_vec(); + let polkavm = include_bytes!( + "../../worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm" + ) + .to_vec(); + let wasm_zst = include_bytes!( + "../../worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm.zst" + ) + .to_vec(); + let wasm = include_bytes!( + "../../worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm" + ) + .to_vec(); + + // Upload the module code and config. + helper.update_version(ProtocolVersion { + major: 2, + minor: 0, + patch: 0, + }); + helper + .upload_modules_and_config( + ProtocolInitializationMethod::SaveContextFromFirstInstance, + vec![ + (BackendModuleKind::PolkaVM, 2, polkavm_zst.clone()), + (BackendModuleKind::PolkaVM, 1, polkavm.clone()), + (BackendModuleKind::Wasm, 2, wasm_zst.clone()), + (BackendModuleKind::Wasm, 1, wasm.clone()), + ], + ) + .await?; + Ok(()) + } + + #[tokio::test] + #[test_log::test] + async fn test_protocol_upgrades() -> Result<()> { + let mut tester = PolymeshTester::new().await?; + let mut users = tester.users(&["ProtocolTester"]).await?.into_iter(); + let mut user = users.next().expect("User not found"); + + // Get the current protocol version from the worker-testing pallet. + let protocol = tester + .api + .query() + .worker_testing() + .current_protocol_version() + .await? + .expect("Current protocol version not found"); + let mut worker_helper = WorkerModulesHelper::new(&tester, protocol.clone()); + + // Enable work session in the worker-testing pallet. + worker_helper + .sudo_call( + tester + .api + .call() + .worker_testing() + .set_enable_work_session(true)?, + ) + .await?; + + // Run the `VerifyVersion` work request. + tester + .api + .call() + .worker_testing() + .test_version(protocol.clone())? + .submit_and_watch(&mut user) + .await? + .wait_finalized() + .await?; + + // Register the testing protocol with the WorkerModules pallet. + worker_helper + .register_protocol( + "Testing", + "Polymesh Worker Protocol Testing", + protocol.version.clone(), + ) + .await?; + + // Upload the v1 protocol modules and config. This only uploads the new modules and config for the new version, and does not change the active protocol version. + upload_v1(&mut worker_helper).await?; + + // Verify that the active protocol version is still the same. + tester + .api + .call() + .worker_testing() + .test_version(protocol.clone())? + .submit_and_watch(&mut user) + .await? + .wait_finalized() + .await?; + + // Change the active protocol version to v1.0.0. + let protocol_v1 = Protocol { + id: protocol.id.clone(), + version: ProtocolVersion { + major: 1, + minor: 0, + patch: 0, + }, + }; + worker_helper + .sudo_call( + tester + .api + .call() + .worker_testing() + .set_protocol_version(protocol_v1.clone())?, + ) + .await?; + + // Verify that the active protocol version is now v1.0.0. + let mut res = tester + .api + .call() + .worker_testing() + .test_version(protocol_v1.clone())? + .submit_and_watch(&mut user) + .await?; + res.ok().await?; + res.wait_finalized().await?; + + // Upload the v2 protocol modules and config. This only uploads the new modules and config for the new version, and does not change the active protocol version. + upload_v2(&mut worker_helper).await?; + + // Verify that the active protocol version is still v1.0.0. + let mut res = tester + .api + .call() + .worker_testing() + .test_version(protocol_v1.clone())? + .submit_and_watch(&mut user) + .await?; + res.ok().await?; + res.wait_finalized().await?; + + // Change the active protocol version to v2.0.0. + let protocol_v2 = Protocol { + id: protocol.id.clone(), + version: ProtocolVersion { + major: 2, + minor: 0, + patch: 0, + }, + }; + worker_helper + .sudo_call( + tester + .api + .call() + .worker_testing() + .set_protocol_version(protocol_v2.clone())?, + ) + .await?; + + // Verify that the active protocol version is now v2.0.0. + let mut res = tester + .api + .call() + .worker_testing() + .test_version(protocol_v2.clone())? + .submit_and_watch(&mut user) + .await?; + res.ok().await?; + res.wait_finalized().await?; + + Ok(()) + } +} diff --git a/pallets/confidential-assets/Cargo.toml b/pallets/confidential-assets/Cargo.toml index fbe0d4b95a..572147b9aa 100644 --- a/pallets/confidential-assets/Cargo.toml +++ b/pallets/confidential-assets/Cargo.toml @@ -14,7 +14,8 @@ pallet-identity = { workspace = true, default-features = false } pallet-transaction-payment = { workspace = true, default-features = false } # Crypto -polymesh-worker-extension = { workspace = true, default-features = false } +polymesh-worker-extension = { workspace = true, default-features = false, optional = true } +polymesh-worker-common = { workspace = true, default-features = false } rand_core = { workspace = true, default-features = false, optional = true } rand_chacha = { workspace = true, default-features = false, optional = true } @@ -22,7 +23,7 @@ rand_chacha = { workspace = true, default-features = false, optional = true } polymesh-dart = { workspace = true, default-features = false, features = [ "backend_bp", "sp-io" ] } -polymesh-worker-protocol-dart-v1 = { workspace = true, default-features = false, features = ["runtime"] } +polymesh-worker-protocol-dart-v1 = { workspace = true, default-features = false } # Substrate codec = { workspace = true, default-features = false, features = ["derive"] } @@ -48,7 +49,22 @@ log = { version = "0.4", default-features = false } equalize = [] default = ["std", "equalize"] -testing = ["polymesh-worker-extension/testing"] +# Use host: worker modules + msm + Hash to Curve +host_workers = [ + "polymesh-worker-extension", + "polymesh-worker-protocol-dart-v1/runtime", +] +# No host functions +no_host_fns = [ + "polymesh-worker-protocol-dart-v1/runtime_only", +] +# Verify proofs in runtime with only these host functions: msm + Hash to Curve +host_curves = [ + "polymesh-worker-protocol-dart-v1/runtime_only", + "polymesh-worker-protocol-dart-v1/use_host_curves", +] + +testing = ["polymesh-worker-extension?/testing"] asm = ["polymesh-dart/asm"] parallel = ["polymesh-dart/parallel"] @@ -71,7 +87,7 @@ std = [ "pallet-transaction-payment/std", "polymesh-runtime-common/std", "polymesh-primitives/std", - "polymesh-worker-extension/std", + "polymesh-worker-extension?/std", "polymesh-worker-protocol-dart-v1/std", "sp-consensus-babe/std", "sp-runtime/std", @@ -85,7 +101,7 @@ runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "pallet-identity/runtime-benchmarks", "polymesh-primitives/runtime-benchmarks", - "polymesh-worker-extension/runtime-benchmarks", + "polymesh-worker-extension?/runtime-benchmarks", "polymesh-worker-protocol-dart-v1/runtime-benchmarks", "sp-runtime/runtime-benchmarks", ] diff --git a/pallets/confidential-assets/src/lib.rs b/pallets/confidential-assets/src/lib.rs index f96cddc54d..42bf2195c6 100644 --- a/pallets/confidential-assets/src/lib.rs +++ b/pallets/confidential-assets/src/lib.rs @@ -55,13 +55,16 @@ use polymesh_dart::{ SettlementCounts, SettlementProof, SettlementRef, FEE_ASSET_ID, }; +use polymesh_worker_common::WorkerSessionId; +#[cfg(feature = "host_workers")] use polymesh_worker_extension::{ - native_polymesh_worker, BackendKind, WorkRequestConfig, WorkerSessionConfig, WorkerSessionId, -}; -use polymesh_worker_protocol_dart_v1::{ - UpdateAssetStateRequest, VerifyDartAssetRequest, PROTOCOL as DART_PROTOCOL, + native_polymesh_worker, BackendKind, WorkRequestConfig, WorkerSessionConfig, }; +#[cfg(feature = "host_workers")] +use polymesh_worker_protocol_dart_v1::PROTOCOL as DART_PROTOCOL; +use polymesh_worker_protocol_dart_v1::{UpdateAssetStateRequest, VerifyDartAssetRequest}; + pub type BalanceOf = <::Currency as Inspect<::AccountId>>::Balance; @@ -2944,32 +2947,47 @@ impl Pallet { } pub fn start_session() { - // Start worker session. - let config = WorkerSessionConfig { - work: WorkRequestConfig { - use_cache: true, - use_thread_pool: false, - }, - init_module: true, - - backends: BackendKind::all_mask(), - } - .to_flags_and_backends(); - let session_id = native_polymesh_worker::start_session(config, DART_PROTOCOL.to_number()); + #[cfg(feature = "host_workers")] + { + // Start worker session. + let config = WorkerSessionConfig { + work: WorkRequestConfig { + use_cache: true, + use_thread_pool: false, + }, + init_module: true, + + backends: BackendKind::all_mask(), + } + .to_flags_and_backends(); + + let session_id = + native_polymesh_worker::start_session(config, DART_PROTOCOL.to_number()); - CurrentWorkerSessionId::::put(session_id); + CurrentWorkerSessionId::::put(session_id); + } } pub fn end_session() { - // Close the batch. - if let Some(session_id) = CurrentWorkerSessionId::::take() { - native_polymesh_worker::end_session(session_id); + #[cfg(feature = "host_workers")] + { + // Close the batch. + if let Some(session_id) = CurrentWorkerSessionId::::take() { + native_polymesh_worker::end_session(session_id); + } } } pub fn submit_and_wait(req: VerifyDartAssetRequest) -> DispatchResult { - req.submit_and_wait(Self::session_id()?) - .map_err(|_| Error::::InvalidProof)?; + #[cfg(feature = "host_workers")] + { + req.submit_and_wait(Self::session_id()?) + .map_err(|_| Error::::InvalidProof)?; + } + #[cfg(not(feature = "host_workers"))] + { + req.do_verify().map_err(|_| Error::::InvalidProof)?; + } Ok(()) } diff --git a/pallets/runtime/common/src/runtime.rs b/pallets/runtime/common/src/runtime.rs index 8e43089e4c..7f368668f2 100644 --- a/pallets/runtime/common/src/runtime.rs +++ b/pallets/runtime/common/src/runtime.rs @@ -863,6 +863,17 @@ macro_rules! misc_pallet_impls { type MaxServiceWeight = MbmServiceWeight; type WeightInfo = polymesh_weights::pallet_migrations::SubstrateWeight; } + + pub type MaxModuleCodeSize = polymesh_primitives::ConstSize<{ 2 * 1024 * 1024 }>; // 2 MiB + pub type MaxModuleContextSize = polymesh_primitives::ConstSize<{ 2 * 1024 * 1024 }>; // 2 MiB + pub type MaxModulesPerConfig = polymesh_primitives::ConstSize<4>; // 4 modules per config + + impl pallet_worker_modules::Config for Runtime { + type WeightInfo = pallet_worker_modules::weights::SubstrateWeight; + type MaxModuleCodeSize = MaxModuleCodeSize; + type MaxModuleContextSize = MaxModuleContextSize; + type MaxModulesPerConfig = MaxModulesPerConfig; + } }; } diff --git a/pallets/runtime/develop/Cargo.toml b/pallets/runtime/develop/Cargo.toml index 2eaf3a81a7..a82873bd66 100644 --- a/pallets/runtime/develop/Cargo.toml +++ b/pallets/runtime/develop/Cargo.toml @@ -40,6 +40,8 @@ pallet-transaction-payment = { workspace = true, default-features = false } pallet-treasury = { workspace = true, default-features = false } pallet-utility = { workspace = true, default-features = false } pallet-precompiles = { workspace = true, default-features = false } +pallet-worker-modules = { workspace = true, default-features = false } +pallet-worker-testing = { workspace = true, default-features = false } polymesh-transaction-payment = { workspace = true, default-features = false } # Others @@ -122,10 +124,27 @@ ci-runtime = [] # Backends u64_backend = ["polymesh-primitives/u64_backend"] +# Use host: worker modules + msm + Hash to Curve +host_workers = [ + "pallet-confidential-assets/host_workers", + "pallet-worker-testing/host_workers", +] +# No host functions +no_host_fns = [ + "pallet-confidential-assets/no_host_fns", + "pallet-worker-testing/no_host_fns", +] +# Verify proofs in runtime with only these host functions: msm + Hash to Curve +host_curves = [ + "pallet-confidential-assets/host_curves", + "pallet-worker-testing/no_host_fns", +] + no_std = [ "polymesh-primitives/no_std", "u64_backend", "pallet-confidential-assets/no_std", + "pallet-worker-testing/no_std", "polymesh-dart/no_std", ] @@ -171,6 +190,8 @@ std = [ "pallet-transaction-payment/std", "polymesh-transaction-payment/std", "pallet-utility/std", + "pallet-worker-modules/std", + "pallet-worker-testing/std", "pallet-validators/std", "pallet-precompiles/std", "polymesh-runtime-common/std", @@ -224,6 +245,8 @@ runtime-benchmarks = [ "polymesh-transaction-payment/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", "pallet-utility/runtime-benchmarks", + "pallet-worker-modules/runtime-benchmarks", + "pallet-worker-testing/runtime-benchmarks", "pallet-validators/runtime-benchmarks", "pallet-precompiles/runtime-benchmarks", "polymesh-runtime-common/runtime-benchmarks", diff --git a/pallets/runtime/develop/src/runtime.rs b/pallets/runtime/develop/src/runtime.rs index 4ffff5e406..d0bbbab317 100644 --- a/pallets/runtime/develop/src/runtime.rs +++ b/pallets/runtime/develop/src/runtime.rs @@ -544,6 +544,12 @@ mod runtime { #[runtime::pallet_index(55)] pub type MultiBlockMigrations = pallet_migrations::Pallet; + #[runtime::pallet_index(60)] + pub type WorkerModules = pallet_worker_modules::Pallet; + + #[runtime::pallet_index(200)] + pub type WorkerTesting = pallet_worker_testing::Pallet; + #[runtime::pallet_index(70)] pub type ConfidentialAssets = pallet_confidential_assets::Pallet; @@ -551,6 +557,10 @@ mod runtime { pub type Revive = pallet_revive::Pallet; } +impl pallet_worker_testing::Config for Runtime { + type WeightInfo = pallet_worker_testing::weights::SubstrateWeight; +} + impl pallet_confidential_assets::Config for Runtime { type Currency = Balances; @@ -600,6 +610,7 @@ mod benches { [pallet_corporate_ballot, CorporateBallot] [pallet_capital_distribution, CapitalDistribution] [pallet_confidential_assets, ConfidentialAssets] + [pallet_worker_modules, WorkerModules] [pallet_external_agents, ExternalAgents] [pallet_relayer, Relayer] [pallet_committee, PolymeshCommittee] diff --git a/pallets/runtime/mainnet/Cargo.toml b/pallets/runtime/mainnet/Cargo.toml index 3d3e7aba70..a085acadf1 100644 --- a/pallets/runtime/mainnet/Cargo.toml +++ b/pallets/runtime/mainnet/Cargo.toml @@ -39,6 +39,7 @@ pallet-transaction-payment = { workspace = true, default-features = false } pallet-treasury = { workspace = true, default-features = false } pallet-utility = { workspace = true, default-features = false } pallet-precompiles = { workspace = true, default-features = false } +pallet-worker-modules = { workspace = true, default-features = false } polymesh-transaction-payment = { workspace = true, default-features = false } # Others @@ -169,6 +170,7 @@ std = [ "polymesh-transaction-payment/std", "pallet-treasury/std", "pallet-utility/std", + "pallet-worker-modules/std", "polymesh-primitives/std", "pallet-precompiles/std", "polymesh-runtime-common/std", @@ -225,6 +227,7 @@ runtime-benchmarks = [ "polymesh-transaction-payment/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", "pallet-utility/runtime-benchmarks", + "pallet-worker-modules/runtime-benchmarks", "pallet-validators/runtime-benchmarks", "pallet-precompiles/runtime-benchmarks", "polymesh-runtime-common/runtime-benchmarks", diff --git a/pallets/runtime/mainnet/src/runtime.rs b/pallets/runtime/mainnet/src/runtime.rs index 86e96e9c8f..75860d752e 100644 --- a/pallets/runtime/mainnet/src/runtime.rs +++ b/pallets/runtime/mainnet/src/runtime.rs @@ -489,6 +489,9 @@ mod runtime { #[runtime::pallet_index(55)] pub type MultiBlockMigrations = pallet_migrations::Pallet; + #[runtime::pallet_index(60)] + pub type WorkerModules = pallet_worker_modules::Pallet; + #[runtime::pallet_index(80)] pub type Revive = pallet_revive::Pallet; } diff --git a/pallets/runtime/testnet/Cargo.toml b/pallets/runtime/testnet/Cargo.toml index 47e02db18b..113ae861fa 100644 --- a/pallets/runtime/testnet/Cargo.toml +++ b/pallets/runtime/testnet/Cargo.toml @@ -40,6 +40,7 @@ pallet-transaction-payment = { workspace = true, default-features = false } pallet-treasury = { workspace = true, default-features = false } pallet-utility = { workspace = true, default-features = false } pallet-precompiles = { workspace = true, default-features = false } +pallet-worker-modules = { workspace = true, default-features = false } polymesh-transaction-payment = { workspace = true, default-features = false } # Others @@ -117,6 +118,19 @@ testing = [] # Backends u64_backend = ["polymesh-primitives/u64_backend"] +# Use host: worker modules + msm + Hash to Curve +host_workers = [ + "pallet-confidential-assets/host_workers", +] +# No host functions +no_host_fns = [ + "pallet-confidential-assets/no_host_fns", +] +# Verify proofs in runtime with only these host functions: msm + Hash to Curve +host_curves = [ + "pallet-confidential-assets/host_curves", +] + migration-dry-run = [] no_std = [ "polymesh-primitives/no_std", @@ -180,6 +194,7 @@ std = [ "polymesh-transaction-payment/std", "pallet-treasury/std", "pallet-utility/std", + "pallet-worker-modules/std", "polymesh-primitives/std", "pallet-precompiles/std", "polymesh-runtime-common/std", @@ -239,6 +254,7 @@ runtime-benchmarks = [ "polymesh-transaction-payment/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", "pallet-utility/runtime-benchmarks", + "pallet-worker-modules/runtime-benchmarks", "pallet-validators/runtime-benchmarks", "polymesh-runtime-common/runtime-benchmarks", ] diff --git a/pallets/runtime/testnet/src/runtime.rs b/pallets/runtime/testnet/src/runtime.rs index 3afa53c6ec..0d33690f43 100644 --- a/pallets/runtime/testnet/src/runtime.rs +++ b/pallets/runtime/testnet/src/runtime.rs @@ -522,6 +522,9 @@ mod runtime { #[runtime::pallet_index(55)] pub type MultiBlockMigrations = pallet_migrations::Pallet; + #[runtime::pallet_index(60)] + pub type WorkerModules = pallet_worker_modules::Pallet; + #[runtime::pallet_index(70)] pub type ConfidentialAssets = pallet_confidential_assets::Pallet; diff --git a/pallets/runtime/tests/Cargo.toml b/pallets/runtime/tests/Cargo.toml index 4f39c8b902..750763cf95 100644 --- a/pallets/runtime/tests/Cargo.toml +++ b/pallets/runtime/tests/Cargo.toml @@ -32,6 +32,7 @@ pallet-transaction-payment = { workspace = true, default-features = false } polymesh-transaction-payment = { workspace = true, default-features = false } pallet-treasury = { workspace = true, default-features = false } pallet-utility = { workspace = true, default-features = false } +pallet-worker-modules = { workspace = true, default-features = false } polymesh-primitives = { workspace = true, default-features = false } pallet-precompiles = { workspace = true, default-features = false } polymesh-precompiles = { workspace = true, default-features = false } diff --git a/pallets/runtime/tests/src/storage.rs b/pallets/runtime/tests/src/storage.rs index bea5c74ff4..f866ddf8fa 100644 --- a/pallets/runtime/tests/src/storage.rs +++ b/pallets/runtime/tests/src/storage.rs @@ -414,6 +414,9 @@ mod runtime { #[runtime::pallet_index(55)] pub type MultiBlockMigrations = pallet_migrations::Pallet; + #[runtime::pallet_index(60)] + pub type WorkerModules = pallet_worker_modules::Pallet; + #[runtime::pallet_index(80)] pub type Revive = pallet_revive::Pallet; diff --git a/pallets/worker-modules/Cargo.toml b/pallets/worker-modules/Cargo.toml new file mode 100644 index 0000000000..f890e6eb8d --- /dev/null +++ b/pallets/worker-modules/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "pallet-worker-modules" +version.workspace = true +authors.workspace = true +license-file.workspace = true +edition.workspace = true +repository.workspace = true +homepage.workspace = true + +[dependencies] +# Common +polymesh-primitives = { workspace = true, default-features = false } + +# Workers +polymesh-worker-extension = { workspace = true, default-features = false } +polymesh-worker-common = { workspace = true, default-features = false, features = ["serde"] } + +# Other +serde = { version = "1.0.104", default-features = false } + +# Substrate +codec = { workspace = true, default-features = false, features = ["derive"] } +scale-info = { workspace = true, default-features = false, features = [ + "derive", +] } +frame-system = { workspace = true, default-features = false } +frame-support = { workspace = true, default-features = false } +sp-std = { workspace = true, default-features = false } +sp-io = { workspace = true, default-features = false } +sp-runtime = { workspace = true, default-features = false } + +# Only Benchmarking +frame-benchmarking = { workspace = true, default-features = false, optional = true } + +log = { version = "0.4", default-features = false } + +[features] +equalize = [] +default = ["std", "equalize"] + +testing = ["polymesh-worker-common/testing"] + +no_std = [] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "frame-benchmarking?/std", + "polymesh-primitives/std", + "polymesh-worker-extension/std", + "polymesh-worker-common/std", + "sp-runtime/std", + "sp-std/std", + "sp-io/std", + "serde/std", +] +runtime-benchmarks = [ + "testing", + "frame-benchmarking/runtime-benchmarks", + "polymesh-primitives/runtime-benchmarks", + "polymesh-worker-extension/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] +try-runtime = [] diff --git a/pallets/worker-modules/src/benchmarking.rs b/pallets/worker-modules/src/benchmarking.rs new file mode 100644 index 0000000000..c87ebd6c95 --- /dev/null +++ b/pallets/worker-modules/src/benchmarking.rs @@ -0,0 +1,96 @@ +use frame_benchmarking::benchmarks; +use frame_support::dispatch::RawOrigin; +use frame_support::pallet_prelude::*; +use sp_std::vec; + +use polymesh_worker_common::{BackendModuleDefinition, BackendModuleKind}; + +use crate::*; + +fn dummy_protocol() -> (ProtocolId, ProtocolMetadata) { + let protocol_id = ProtocolId(1); + let metadata = ProtocolMetadata { + protocol_name: "Test Protocol" + .as_bytes() + .to_vec() + .try_into() + .expect("Failed to convert protocol name to BoundedVec"), + protocol_description: "A test protocol for benchmarking" + .as_bytes() + .to_vec() + .try_into() + .expect("Failed to convert protocol description to BoundedVec"), + protocol_version: ProtocolVersion::default(), + }; + (protocol_id, metadata) +} + +fn register_dummy_protocol() -> Result { + let (protocol_id, metadata) = dummy_protocol(); + let protocol = Protocol { + id: protocol_id, + version: ProtocolVersion::default(), + }; + Pallet::::register_protocol(RawOrigin::Root.into(), protocol_id, metadata)?; + + Ok(protocol) +} + +benchmarks! { + register_protocol { + let (protocol_id, metadata) = dummy_protocol(); + }: _(RawOrigin::Root, protocol_id, metadata) + + upload_protocol_module_code { + let protocol = register_dummy_protocol::().expect("Failed to register dummy protocol"); + let code = vec![0u8; T::MaxModuleCodeSize::get() as usize].try_into().expect("Failed to convert code to BoundedVec"); // dummy code + }: _(RawOrigin::Root, protocol, code) + + upload_protocol_module_context { + let protocol = register_dummy_protocol::().expect("Failed to register dummy protocol"); + let context = vec![0u8; T::MaxModuleContextSize::get() as usize].try_into().expect("Failed to convert context to BoundedVec"); // dummy context + }: _(RawOrigin::Root, protocol, context) + + upload_protocol_module_config { + let m in 1 .. T::MaxModulesPerConfig::get() as u32; + + // Register a dummy protocol. + let protocol = register_dummy_protocol::().expect("Failed to register dummy protocol"); + + // Upload dummy code for each module. + let mut modules = vec![]; + for idx in 0..(m-1) { + let code = vec![idx as u8; 1024]; // dummy code + let code_hash = code.using_encoded(sp_io::hashing::blake2_256); + Pallet::::upload_protocol_module_code(RawOrigin::Root.into(), protocol, code.try_into().expect("Failed to convert code to BoundedVec")) + .expect("Failed to upload dummy module code"); + modules.push(BackendModuleDefinition { + module_kind: BackendModuleKind::Wasm, + module_version: 1, + code_hash, + }); + } + + // Add a native module to the config. + let native_code = protocol.encode(); + let native_code_hash = blake2_256(&native_code); + modules.push(BackendModuleDefinition { + module_kind: BackendModuleKind::Native, + module_version: 1, + code_hash: native_code_hash, + }); + + // Create a dummy protocol module config with the uploaded modules. + let config = ProtocolModuleConfig { + protocol, + initialization_method: ProtocolInitializationMethod::ContextData( + vec![0u8; T::MaxModuleContextSize::get() as usize] + .try_into() + .expect("Failed to convert context to BoundedVec"), // dummy context + ), + modules: modules + .try_into() + .expect("Failed to convert backends to BoundedVec"), + }; + }: _(RawOrigin::Root, config) +} diff --git a/pallets/worker-modules/src/lib.rs b/pallets/worker-modules/src/lib.rs new file mode 100644 index 0000000000..8c8081cd52 --- /dev/null +++ b/pallets/worker-modules/src/lib.rs @@ -0,0 +1,502 @@ +// This file is part of the Polymesh distribution (https://github.com/PolymeshAssociation/Polymesh). +// Copyright (c) 2026 Polymesh + +//! # Worker Modules Pallet +//! +//! This pallet manages updating worker modules for different protocols supported by the Polymesh Worker extension. +//! +//! +//! + +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen}; +use frame_support::traits::ConstU32; +use frame_support::{dispatch::DispatchResult, ensure, traits::Get, weights::Weight, BoundedVec}; +use frame_system::pallet_prelude::*; +use scale_info::TypeInfo; +use sp_io::hashing::blake2_256; +use sp_runtime::{Deserialize, Serialize}; +use sp_std::vec::Vec; + +use polymesh_primitives::GetExtra; +use polymesh_worker_common::{ + BackendCodeHash, BackendContextHash, BackendModuleDefinition, BackendModuleKind, Protocol, + ProtocolId, ProtocolModuleConfigHash, ProtocolVersion, +}; +use polymesh_worker_extension::worker_modules_legacy_code_key; + +#[cfg(feature = "runtime-benchmarks")] +pub mod benchmarking; + +pub mod weights; + +pub trait WeightInfo { + fn register_protocol() -> Weight; + fn upload_protocol_module_code() -> Weight; + fn upload_protocol_module_context() -> Weight; + fn upload_protocol_module_config(m: u32) -> Weight; +} + +/// Protocol metadata for a protocol supported by the Polymesh Worker extension. +#[derive( + Clone, + Encode, + Decode, + Serialize, + Deserialize, + PartialEq, + Eq, + Default, + Debug, + MaxEncodedLen, + TypeInfo, + DecodeWithMemTracking +)] +pub struct ProtocolMetadata { + /// The protocol name. + pub protocol_name: BoundedVec>, + /// The protocol description. + pub protocol_description: BoundedVec>, + /// The protocol version. + pub protocol_version: ProtocolVersion, +} + +/// Protocol initialization method. +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + MaxEncodedLen, + TypeInfo, + DecodeWithMemTracking +)] +#[scale_info(skip_type_params(MaxModuleContextSize))] +pub enum ProtocolInitializationMethod> { + /// No initialization is needed, the module can be used directly after loading. + NoInitializationNeeded, + /// Initialize with no context. + InitializeNoContext, + /// Initialize the first module instance without context and save the context for faster initialization of subsequent instances. + SaveContextFromFirstInstance, + /// The module needs to be initialized with the given context data, which is generated by the runtime and passed to the host for module initialization. + ContextData(BoundedVec), + /// The module needs to be initialized with the given context hash, which is used to load the initialization context data from the cache or from Substrate on-chain storage. + ContextHash(BackendContextHash), +} + +/// A protocol's module configuration, which contains the list of backends and their corresponding module definitions (code hash, context hash, etc). +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + MaxEncodedLen, + TypeInfo, + DecodeWithMemTracking +)] +#[scale_info(skip_type_params(MaxModuleContextSize, MaxModulesPerConfig))] +pub struct ProtocolModuleConfig, MaxModulesPerConfig: Get> { + /// The protocol id and version are included here to make sure the config hash includes the protocol information, which is used for caching the loaded module. + pub protocol: Protocol, + /// The initialization method for the protocol module, which is used to determine how to initialize the module after loading. + pub initialization_method: ProtocolInitializationMethod, + /// The list of backend module definitions for the protocol, which contains the module code hash and context hash for each backend. + pub modules: BoundedVec, +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::without_storage_info] + pub struct Pallet(_); + + /// Configuration trait. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Confidential asset pallet weights. + type WeightInfo: WeightInfo; + + /// Maximum module code size in bytes. + type MaxModuleCodeSize: GetExtra; + + /// Maximum module context size in bytes. + type MaxModuleContextSize: GetExtra; + + /// Maximum number of modules per protocol config. + type MaxModulesPerConfig: GetExtra; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A new protocol has been registered. + ProtocolRegistered { + /// The ID of the registered protocol. + protocol_id: ProtocolId, + /// The metadata of the registered protocol. + metadata: ProtocolMetadata, + }, + /// Protocol module code has been uploaded. + ProtocolModuleCodeUploaded { + /// The protocol the module code belongs to. + protocol: Protocol, + /// The uploaded module code. + code: BoundedVec, + /// The hash of the uploaded module code. + code_hash: BackendCodeHash, + }, + /// Protocol module context has been uploaded. + ProtocolModuleContextUploaded { + /// The protocol the module context belongs to. + protocol: Protocol, + /// The uploaded module context. + context: BoundedVec, + /// The hash of the uploaded module context. + context_hash: BackendContextHash, + }, + /// Protocol module config has been uploaded. + ProtocolModuleConfigUploaded { + /// The protocol the module config belongs to. + protocol: Protocol, + /// The uploaded module config. + config: ProtocolModuleConfig, + /// The hash of the uploaded module config. + config_hash: ProtocolModuleConfigHash, + }, + } + + #[pallet::error] + pub enum Error { + /// The protocol is already registered. + ProtocolAlreadyRegistered, + /// The protocol is not registered. + ProtocolNotRegistered, + /// The protocol metadata is invalid. + InvalidProtocolMetadata, + /// The protocol module code already exists. + ProtocolModuleCodeAlreadyExists, + /// The protocol module context already exists. + ProtocolModuleContextAlreadyExists, + /// The protocol module config already exists. + ProtocolModuleConfigAlreadyExists, + /// Module code missing for the given code hash. + ModuleCodeMissing, + /// Module context missing for the given context hash. + ModuleContextMissing, + } + + /// Protocol metadata storage item, which maps ProtocolId => ProtocolMetadata. + #[pallet::storage] + pub type Metadata = + StorageMap<_, Blake2_128Concat, ProtocolId, ProtocolMetadata, OptionQuery>; + + /// Protocol config hash storage item, which maps Protocol => ProtocolModuleConfigHash. + #[pallet::storage] + pub type ProtocolConfigHash = + StorageMap<_, Identity, Protocol, ProtocolModuleConfigHash, OptionQuery>; + + /// Protocol config storage item, which double maps (Protocol, ProtocolModuleConfigHash) => ProtocolModuleConfig. + #[pallet::storage] + pub type ProtocolConfig = StorageDoubleMap< + _, + Identity, + Protocol, + Identity, + ProtocolModuleConfigHash, + ProtocolModuleConfig, + OptionQuery, + >; + + /// Protocol module code storage item, which double maps (Protocol, BackendCodeHash) => Vec. + #[pallet::storage] + pub type ProtocolCode = StorageDoubleMap< + _, + Identity, + Protocol, + Identity, + BackendCodeHash, + BoundedVec, + OptionQuery, + >; + + /// Protocol context storage item, which double maps (Protocol, BackendContextHash) => Vec. + #[pallet::storage] + pub type ProtocolContext = StorageDoubleMap< + _, + Identity, + Protocol, + Identity, + BackendContextHash, + BoundedVec, + OptionQuery, + >; + + #[pallet::genesis_config] + #[derive(frame_support::DefaultNoBound)] + pub struct GenesisConfig { + pub protocols: Vec<(ProtocolId, ProtocolMetadata)>, + pub _config: sp_std::marker::PhantomData, + } + + #[pallet::genesis_build] + impl BuildGenesisConfig for GenesisConfig { + fn build(&self) { + for (protocol_id, metadata) in &self.protocols { + Metadata::::insert(protocol_id, metadata.clone()); + } + } + } + + #[pallet::call] + impl Pallet { + /// Register a new protocol. + /// + /// # Arguments + /// * `origin` - The origin of the call, must be root. + /// * `protocol_id` - The ID of the protocol to register. + /// * `metadata` - The metadata of the protocol to register. + /// + /// # Errors + /// * `DispatchError::BadOrigin` - If the origin is not root. + /// * `Error::::ProtocolAlreadyRegistered` - If the protocol is already registered. + /// * `Error::::InvalidProtocolMetadata` - If the protocol metadata is invalid. + #[pallet::call_index(0)] + #[pallet::weight(::WeightInfo::register_protocol())] + pub fn register_protocol( + origin: OriginFor, + protocol_id: ProtocolId, + metadata: ProtocolMetadata, + ) -> DispatchResult { + ensure_root(origin)?; + + // Ensure the protocol is not already registered. + ensure!( + !Metadata::::contains_key(&protocol_id), + Error::::ProtocolAlreadyRegistered + ); + + // Ensure the protocol metadata is valid. + ensure!( + !metadata.protocol_name.is_empty(), + Error::::InvalidProtocolMetadata + ); + + // Store the protocol metadata. + Metadata::::insert(&protocol_id, &metadata); + + // Emit an event. + Self::deposit_event(Event::ProtocolRegistered { + protocol_id, + metadata, + }); + + Ok(()) + } + + /// Upload protocol module code. + /// + /// # Arguments + /// * `origin` - The origin of the call, must be root. + /// * `protocol` - The protocol to upload the module code for. + /// * `code` - The module code to upload. + /// * `module_version` - The version of the module being uploaded. + /// + /// # Errors + /// * `DispatchError::BadOrigin` - If the origin is not root. + /// * `Error::::ProtocolNotRegistered` - If the protocol is not registered. + /// * `Error::::ProtocolModuleCodeAlreadyExists` - If the protocol module code already exists. + #[pallet::call_index(1)] + #[pallet::weight(::WeightInfo::upload_protocol_module_code())] + pub fn upload_protocol_module_code( + origin: OriginFor, + protocol: Protocol, + code: BoundedVec, + module_version: u32, + ) -> DispatchResult { + ensure_root(origin)?; + + // Ensure the protocol is registered. + ensure!( + Metadata::::contains_key(&protocol.id), + Error::::ProtocolNotRegistered + ); + + let code_hash = blake2_256(&code); + if module_version > 1 { + // Ensure the code hash is not already stored for the protocol. + ensure!( + !ProtocolCode::::contains_key(&protocol, &code_hash), + Error::::ProtocolModuleCodeAlreadyExists + ); + + // Store the protocol module code. + ProtocolCode::::insert(&protocol, &code_hash, &code); + } else { + // Store the legacy protocol module code under the generated key. + let key = worker_modules_legacy_code_key(protocol, code_hash); + + // Ensure the code hash is not already stored for the protocol. + ensure!( + !sp_io::storage::exists(&key), + Error::::ProtocolModuleCodeAlreadyExists + ); + + sp_io::storage::set(&key, &code); + } + + // Emit an event. + Self::deposit_event(Event::ProtocolModuleCodeUploaded { + protocol, + code, + code_hash, + }); + + Ok(()) + } + + /// Upload protocol module context. + /// + /// # Arguments + /// * `origin` - The origin of the call, must be root. + /// * `protocol` - The protocol to upload the module context for. + /// * `context` - The module context to upload. + /// + /// # Errors + /// * `DispatchError::BadOrigin` - If the origin is not root. + /// * `Error::::ProtocolNotRegistered` - If the protocol is not registered. + /// * `Error::::ProtocolModuleContextAlreadyExists` - If the protocol module context already exists. + #[pallet::call_index(2)] + #[pallet::weight(::WeightInfo::upload_protocol_module_context())] + pub fn upload_protocol_module_context( + origin: OriginFor, + protocol: Protocol, + context: BoundedVec, + ) -> DispatchResult { + ensure_root(origin)?; + + // Ensure the protocol is registered. + ensure!( + Metadata::::contains_key(&protocol.id), + Error::::ProtocolNotRegistered + ); + + let context_hash = blake2_256(&context); + // Ensure the context hash is not already stored for the protocol. + ensure!( + !ProtocolContext::::contains_key(&protocol, &context_hash), + Error::::ProtocolModuleContextAlreadyExists + ); + + // Store the protocol module context. + ProtocolContext::::insert(&protocol, &context_hash, &context); + + // Emit an event. + Self::deposit_event(Event::ProtocolModuleContextUploaded { + protocol, + context, + context_hash, + }); + Ok(()) + } + + /// Upload protocol module config. + /// + /// # Arguments + /// * `origin` - The origin of the call, must be root. + /// * `config` - The module config to upload. + /// + /// # Errors + /// * `DispatchError::BadOrigin` - If the origin is not root. + /// * `Error::::ProtocolNotRegistered` - If the protocol is not registered. + /// * `Error::::ProtocolModuleConfigAlreadyExists` - If the protocol module config already exists. + #[pallet::call_index(3)] + #[pallet::weight(::WeightInfo::upload_protocol_module_config(config.modules.len() as u32))] + pub fn upload_protocol_module_config( + origin: OriginFor, + config: ProtocolModuleConfig, + ) -> DispatchResult { + ensure_root(origin)?; + + let protocol = config.protocol.clone(); + // Ensure the protocol is registered. + ensure!( + Metadata::::contains_key(&protocol.id), + Error::::ProtocolNotRegistered + ); + + // Ensure the context hashes exist for the protocol. + match &config.initialization_method { + ProtocolInitializationMethod::ContextHash(context_hash) => { + ensure!( + ProtocolContext::::contains_key(&protocol, &context_hash), + Error::::ModuleContextMissing + ); + } + _ => {} + } + + // Ensure all module code hashes exist for the protocol. + for module in &config.modules { + if module.module_kind == BackendModuleKind::Native { + // Encode the protocol as the native code (so each protocol version has a unique native code hash). + let native_code: BoundedVec = protocol + .encode() + .try_into() + .map_err(|_| Error::::ModuleCodeMissing)?; + let native_code_hash = blake2_256(&native_code); + ensure!( + module.code_hash == native_code_hash, + Error::::ModuleCodeMissing + ); + // Store the native code for the protocol. + ProtocolCode::::insert(&protocol, &native_code_hash, native_code); + } else if module.module_version > 1 { + ensure!( + ProtocolCode::::contains_key(&protocol, &module.code_hash), + Error::::ModuleCodeMissing + ); + } else { + // Handle legacy protocol module code. + let key = worker_modules_legacy_code_key(protocol, module.code_hash); + ensure!( + !sp_io::storage::exists(&key), + Error::::ProtocolModuleCodeAlreadyExists + ); + } + } + + let config_hash = config.using_encoded(blake2_256); + // Ensure the config hash is not already stored for the protocol. + ensure!( + !ProtocolConfig::::contains_key(&protocol, &config_hash), + Error::::ProtocolModuleConfigAlreadyExists + ); + + // Store the protocol module config. + ProtocolConfig::::insert(&protocol, &config_hash, &config); + + // Point the protocol to the latest config hash for the protocol. + // This allows updating the protocol config. + ProtocolConfigHash::::insert(&protocol, &config_hash); + + // Emit an event. + Self::deposit_event(Event::ProtocolModuleConfigUploaded { + protocol, + config, + config_hash, + }); + Ok(()) + } + } +} diff --git a/pallets/worker-modules/src/weights.rs b/pallets/worker-modules/src/weights.rs new file mode 100644 index 0000000000..35bd53e37b --- /dev/null +++ b/pallets/worker-modules/src/weights.rs @@ -0,0 +1,83 @@ +// This file is part of the Polymesh distribution (https://github.com/PolymeshAssociation/Polymesh). +// Copyright (c) 2023 Polymesh + +//! Autogenerated weights for pallet_worker_modules +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 54.0.0 +//! DATE: 2026-06-23, STEPS: `100`, REPEAT: 4, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: , WASM-EXECUTION: Compiled, CHAIN: None, DB CACHE: 512 +//! HOSTNAME: `trance`, CPU: `AMD Ryzen 9 9950X3D 16-Core Processor` + +// Executed Command: +// target/release/polymesh +// benchmark +// pallet +// --min-duration=10 +// -s +// 100 +// -r +// 4 +// -p=pallet_worker_modules +// -e=* +// --db-cache +// 512 +// --heap-pages +// 4096 +// --output +// ./ +// --template +// .maintain/crate-weight-template.hbs + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use polymesh_primitives::{RocksDbWeight as DbWeight, Weight}; + +/// Weights for pallet_worker_modules using the Substrate node and recommended hardware. +pub struct SubstrateWeight; +impl crate::WeightInfo for SubstrateWeight { + // Storage: `WorkerModules::Metadata` (r:1 w:1) + // Proof: `WorkerModules::Metadata` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn register_protocol() -> Weight { + // Minimum execution time: 6_623 nanoseconds. + Weight::from_parts(7_244_000, 0) + .saturating_add(DbWeight::get().reads(1)) + .saturating_add(DbWeight::get().writes(1)) + } + // Storage: `WorkerModules::Metadata` (r:1 w:0) + // Proof: `WorkerModules::Metadata` (`max_values`: None, `max_size`: None, mode: `Measured`) + // Storage: `WorkerModules::ProtocolModuleCode` (r:1 w:1) + // Proof: `WorkerModules::ProtocolModuleCode` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn upload_protocol_module_code() -> Weight { + // Minimum execution time: 2_923_264 nanoseconds. + Weight::from_parts(3_025_248_000, 0) + .saturating_add(DbWeight::get().reads(2)) + .saturating_add(DbWeight::get().writes(1)) + } + // Storage: `WorkerModules::Metadata` (r:1 w:0) + // Proof: `WorkerModules::Metadata` (`max_values`: None, `max_size`: None, mode: `Measured`) + // Storage: `WorkerModules::ProtocolContext` (r:1 w:1) + // Proof: `WorkerModules::ProtocolContext` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn upload_protocol_module_context() -> Weight { + // Minimum execution time: 2_926_880 nanoseconds. + Weight::from_parts(3_055_737_000, 0) + .saturating_add(DbWeight::get().reads(2)) + .saturating_add(DbWeight::get().writes(1)) + } + // Storage: `WorkerModules::Metadata` (r:1 w:0) + // Proof: `WorkerModules::Metadata` (`max_values`: None, `max_size`: None, mode: `Measured`) + // Storage: `WorkerModules::ProtocolConfig` (r:1 w:1) + // Proof: `WorkerModules::ProtocolConfig` (`max_values`: None, `max_size`: None, mode: `Measured`) + // Storage: `WorkerModules::ProtocolModuleCode` (r:4 w:0) + // Proof: `WorkerModules::ProtocolModuleCode` (`max_values`: None, `max_size`: None, mode: `Measured`) + // Storage: `WorkerModules::ProtocolConfigHash` (r:0 w:1) + // Proof: `WorkerModules::ProtocolConfigHash` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// The range of component `m` is `[1, 4]`. + fn upload_protocol_module_config(m: u32) -> Weight { + // Minimum execution time: 3_193_910 nanoseconds. + Weight::from_parts(3_336_378_600, 0) + .saturating_add(DbWeight::get().reads(2)) + .saturating_add(DbWeight::get().reads((1_u64).saturating_mul(m.into()))) + .saturating_add(DbWeight::get().writes(2)) + } +} diff --git a/pallets/worker-modules/worker-testing/Cargo.toml b/pallets/worker-modules/worker-testing/Cargo.toml new file mode 100644 index 0000000000..926fca3d3f --- /dev/null +++ b/pallets/worker-modules/worker-testing/Cargo.toml @@ -0,0 +1,76 @@ +[package] +name = "pallet-worker-testing" +version = "0.1.0" +authors = ["Polymesh Labs"] +edition = "2021" + +[dependencies] +# Common +polymesh-primitives = { workspace = true, default-features = false } +polymesh-runtime-common = { workspace = true, default-features = false } + +# Polymesh worker extension +polymesh-worker-extension = { workspace = true, default-features = false, optional = true } +polymesh-worker-common = { workspace = true, default-features = false, features = ["serde"] } + +# Testing protocol +polymesh-worker-protocol-testing = { workspace = true, default-features = false } + +# Substrate +codec = { workspace = true, default-features = false, features = ["derive"] } +scale-info = { workspace = true, default-features = false, features = [ + "derive", +] } +frame-system = { workspace = true, default-features = false } +frame-support = { workspace = true, default-features = false } +sp-std = { workspace = true, default-features = false } +sp-io = { workspace = true, default-features = false } +sp-runtime = { workspace = true, default-features = false } + +# Only Benchmarking +frame-benchmarking = { workspace = true, default-features = false, optional = true } + +log = { version = "0.4", default-features = false } + +[features] +equalize = [] +default = ["std", "equalize"] + +# Use host: worker modules +host_workers = [ + "polymesh-worker-extension", + "polymesh-worker-protocol-testing/runtime", +] +# No host functions +no_host_fns = [ + "polymesh-worker-protocol-testing/runtime_only", +] + +testing = ["polymesh-worker-extension?/testing", "polymesh-worker-common/testing"] + +no_std = [ + "polymesh-worker-protocol-testing/no_std" +] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "frame-benchmarking?/std", + "polymesh-runtime-common/std", + "polymesh-primitives/std", + "polymesh-worker-extension?/std", + "polymesh-worker-common/std", + "polymesh-worker-protocol-testing/std", + "sp-runtime/std", + "sp-std/std", + "sp-io/std", +] +runtime-benchmarks = [ + "testing", + "frame-benchmarking/runtime-benchmarks", + "polymesh-primitives/runtime-benchmarks", + "polymesh-worker-extension?/runtime-benchmarks", + "polymesh-worker-protocol-testing/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] +try-runtime = [] diff --git a/pallets/worker-modules/worker-testing/src/lib.rs b/pallets/worker-modules/worker-testing/src/lib.rs new file mode 100644 index 0000000000..141649b328 --- /dev/null +++ b/pallets/worker-modules/worker-testing/src/lib.rs @@ -0,0 +1,222 @@ +// This file is part of the Polymesh distribution (https://github.com/PolymeshAssociation/Polymesh). +// Copyright (c) 2023 Polymesh + +//! # Confidential assets Pallet +//! +//! The Confidential Assets pallet provides sender, receiver, asset and value confidentiality. +//! +//! ## Overview +//! +//! These pallets call out to the [Polymesh DART library](https://github.com/PolymeshAssociation/polymesh-dart) +//! which implements the ZK-proofs for DART. +//! +//! + +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_prelude::DispatchError; +use frame_support::{dispatch::DispatchResult, weights::Weight}; +use frame_system::pallet_prelude::*; + +use polymesh_worker_common::{Protocol, WorkerSessionId}; +#[cfg(feature = "host_workers")] +use polymesh_worker_extension::{ + native_polymesh_worker, BackendKind, WorkRequestConfig, WorkerSessionConfig, +}; +use polymesh_worker_protocol_testing::{ + TestWorkRequest, TestWorkResponse, VerifyVersionRequest, PROTOCOL as TEST_PROTOCOL, +}; + +pub mod weights; + +pub trait WeightInfo { + fn test_version() -> Weight; + fn submit_work_request() -> Weight; + fn set_protocol_version() -> Weight; + fn set_enable_work_session() -> Weight; + fn on_init() -> Weight; +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use polymesh_worker_common::ProtocolError; + use polymesh_worker_protocol_testing::{TestWorkRequest, TestWorkResponse}; + + #[pallet::pallet] + #[pallet::without_storage_info] + pub struct Pallet(_); + + /// Configuration trait. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Confidential asset pallet weights. + type WeightInfo: WeightInfo; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + TestingProtocolTask { + request: TestWorkRequest, + result: Result, + }, + } + + #[pallet::error] + pub enum Error { + /// The worker session is not initialized. + NoSession, + } + + /// The WorkerSessionId for the current block. + #[pallet::storage] + pub(crate) type CurrentWorkerSessionId = + StorageValue<_, WorkerSessionId, OptionQuery>; + + /// Enable work session per block. This is used for testing purpose only. + #[pallet::storage] + pub(crate) type EnableWorkSession = StorageValue<_, bool, ValueQuery>; + + /// The current testing protocol version. + #[pallet::storage] + pub(crate) type CurrentProtocolVersion = StorageValue<_, Protocol, OptionQuery>; + + #[pallet::genesis_config] + #[derive(frame_support::DefaultNoBound)] + pub struct GenesisConfig { + #[serde(skip)] + pub _config: sp_std::marker::PhantomData, + } + + #[pallet::genesis_build] + impl BuildGenesisConfig for GenesisConfig { + fn build(&self) { + CurrentProtocolVersion::::put(TEST_PROTOCOL); + EnableWorkSession::::put(false); + } + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_n: BlockNumberFor) -> Weight { + Self::init_block() + } + + fn on_finalize(_n: BlockNumberFor) { + Self::finalize_block(); + } + } + + #[pallet::call] + impl Pallet { + /// Test the protocol version in the `Testing` protocol module. + #[pallet::call_index(0)] + #[pallet::weight(::WeightInfo::test_version())] + pub fn test_version(origin: OriginFor, protocol: Protocol) -> DispatchResult { + ensure_signed(origin)?; + + // Test verify protocol version work request. + let request = TestWorkRequest::VerifyVersion(VerifyVersionRequest { protocol }); + Self::session_submit_work_request(request)?; + + Ok(()) + } + + /// Set what version of the testing protocol. + #[pallet::call_index(1)] + #[pallet::weight(::WeightInfo::set_protocol_version())] + pub fn set_protocol_version(origin: OriginFor, protocol: Protocol) -> DispatchResult { + ensure_root(origin)?; + CurrentProtocolVersion::::put(protocol); + Ok(()) + } + + /// Set whether to enable work session per block. This is used for testing purpose only. + #[pallet::call_index(2)] + #[pallet::weight(::WeightInfo::set_enable_work_session())] + pub fn set_enable_work_session(origin: OriginFor, enable: bool) -> DispatchResult { + ensure_root(origin)?; + EnableWorkSession::::put(enable); + Ok(()) + } + + /// Submit a work request to the worker session. + #[pallet::call_index(3)] + #[pallet::weight(::WeightInfo::submit_work_request())] + pub fn submit_work_request( + origin: OriginFor, + request: TestWorkRequest, + ) -> DispatchResult { + ensure_signed(origin)?; + Self::session_submit_work_request(request)?; + Ok(()) + } + } +} + +impl Pallet { + pub fn init_block() -> Weight { + if EnableWorkSession::::get() { + Self::start_session(); + } + ::WeightInfo::on_init() + } + + pub fn finalize_block() { + Self::end_session(); + } + + pub fn start_session() { + #[cfg(feature = "host_workers")] + { + // Start worker session. + let config = WorkerSessionConfig { + work: WorkRequestConfig { + use_cache: true, + use_thread_pool: false, + }, + init_module: true, + + backends: BackendKind::all_mask(), + } + .to_flags_and_backends(); + let protocol = CurrentProtocolVersion::::get().unwrap_or(TEST_PROTOCOL); + let session_id = native_polymesh_worker::start_session(config, protocol.to_number()); + + CurrentWorkerSessionId::::put(session_id); + } + } + + pub fn session_submit_work_request( + request: TestWorkRequest, + ) -> Result, DispatchError> { + #[cfg(feature = "host_workers")] + let result = { + let session_id = CurrentWorkerSessionId::::get().ok_or(Error::::NoSession)?; + request.clone().session_execute_and_wait(session_id) + }; + #[cfg(not(feature = "host_workers"))] + let result = request.clone().execute(); + + Self::deposit_event(Event::TestingProtocolTask { + request, + result: result.clone(), + }); + + Ok(result.ok()) + } + + pub fn end_session() { + #[cfg(feature = "host_workers")] + { + // Close the batch. + if let Some(session_id) = CurrentWorkerSessionId::::take() { + native_polymesh_worker::end_session(session_id); + } + } + } +} diff --git a/pallets/worker-modules/worker-testing/src/weights.rs b/pallets/worker-modules/worker-testing/src/weights.rs new file mode 100644 index 0000000000..fc2ebd3e3b --- /dev/null +++ b/pallets/worker-modules/worker-testing/src/weights.rs @@ -0,0 +1,53 @@ +// This file is part of the Polymesh distribution (https://github.com/PolymeshAssociation/Polymesh). +// Copyright (c) 2023 Polymesh + +//! Autogenerated weights for pallet_confidential_assets +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 54.0.0 +//! DATE: 2026-05-12, STEPS: `100`, REPEAT: 5, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: , WASM-EXECUTION: Compiled, CHAIN: None, DB CACHE: 512 +//! HOSTNAME: `ubuntu-8gb-hel1-bench`, CPU: `AMD EPYC-Milan Processor` + +// Executed Command: +// ./target/release/polymesh +// benchmark +// pallet +// -s +// 100 +// -r +// 5 +// -p=pallet_confidential_assets +// -e=* +// --heap-pages +// 4096 +// --db-cache +// 512 +// --output +// ./pallets/confidential-assets/src/weights.rs +// --template +// ./.maintain/crate-weight-template.hbs + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use polymesh_primitives::{RocksDbWeight as DbWeight, Weight}; + +/// Weights for pallet_confidential_assets using the Substrate node and recommended hardware. +pub struct SubstrateWeight; +impl crate::WeightInfo for SubstrateWeight { + fn test_version() -> Weight { + Weight::zero() + } + fn submit_work_request() -> Weight { + Weight::zero() + } + fn set_protocol_version() -> Weight { + Weight::zero() + } + fn set_enable_work_session() -> Weight { + Weight::zero() + } + fn on_init() -> Weight { + Weight::zero() + } +} diff --git a/src/service.rs b/src/service.rs index 1d586e8979..d23d38e98e 100644 --- a/src/service.rs +++ b/src/service.rs @@ -122,13 +122,19 @@ impl RuntimeApiCollection for Api where { } -/// Host functions available to the runtime. -#[cfg(not(feature = "runtime-benchmarks"))] -pub type HostFunctions = ( - sp_io::SubstrateHostFunctions, +#[cfg(feature = "host_workers")] +type PolymeshHostFunctions = ( polymesh_worker_extension::native_polymesh_worker::HostFunctions, polymesh_native_crypto::HostFunctions, ); +#[cfg(feature = "no_host_fns")] +type PolymeshHostFunctions = (); +#[cfg(feature = "host_curves")] +type PolymeshHostFunctions = polymesh_native_crypto::HostFunctions; + +/// Host functions available to the runtime. +#[cfg(not(feature = "runtime-benchmarks"))] +pub type HostFunctions = (sp_io::SubstrateHostFunctions, PolymeshHostFunctions); /// Host functions available to the runtime. #[cfg(feature = "runtime-benchmarks")] @@ -136,8 +142,7 @@ pub type HostFunctions = ( sp_io::SubstrateHostFunctions, frame_benchmarking::benchmarking::HostFunctions, polymesh_primitives::crypto::native_schnorrkel::HostFunctions, - polymesh_worker_extension::native_polymesh_worker::HostFunctions, - polymesh_native_crypto::HostFunctions, + PolymeshHostFunctions, ); /// A specialized `WasmExecutor` intended to use across substrate node. It provides all required HostFunctions. diff --git a/worker/build_polkavm.sh b/worker/build_polkavm.sh index 14f48a8e34..39ed4fa22c 100755 --- a/worker/build_polkavm.sh +++ b/worker/build_polkavm.sh @@ -13,7 +13,7 @@ rm -f "$output_path" "$elf_path" echo "> Building: '$crate' (-> $output_path)" RUSTFLAGS="--remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C codegen-units=1" \ -cargo build \ +cargo rustc --crate-type cdylib \ -Z build-std=core,alloc \ --target $TARGET_JSON_PATH \ --no-default-features \ diff --git a/worker/build_polkavm_testing.sh b/worker/build_polkavm_testing.sh index 510855e5be..e79f2eba7e 100755 --- a/worker/build_polkavm_testing.sh +++ b/worker/build_polkavm_testing.sh @@ -13,7 +13,7 @@ rm -f "$output_path" "$elf_path" echo "> Building: '$crate' (-> $output_path)" RUSTFLAGS="--remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C codegen-units=1" \ -cargo build \ +cargo rustc --crate-type cdylib \ -Z build-std=core,alloc \ --target $TARGET_JSON_PATH \ --no-default-features \ diff --git a/worker/build_wasm.sh b/worker/build_wasm.sh index dc0ceeb363..2cbea2dc05 100755 --- a/worker/build_wasm.sh +++ b/worker/build_wasm.sh @@ -13,7 +13,7 @@ echo "> Building: '$crate' (-> $output_path)" #RUSTFLAGS="-C target-feature=+simd128,+wide-arithmetic --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ RUSTFLAGS="-C target-feature=+simd128 --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ - cargo build \ + cargo rustc --crate-type cdylib \ --target=$target \ --no-default-features \ --features wasm \ diff --git a/worker/build_wasm_testing.sh b/worker/build_wasm_testing.sh index be26f1b534..90b0807489 100755 --- a/worker/build_wasm_testing.sh +++ b/worker/build_wasm_testing.sh @@ -13,7 +13,7 @@ echo "> Building: '$crate' (-> $output_path)" #RUSTFLAGS="-C target-feature=+simd128,+wide-arithmetic --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ RUSTFLAGS="-C target-feature=+simd128 --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ - cargo build \ + cargo rustc --crate-type cdylib \ --target=$target \ --no-default-features \ --features wasm,testing \ diff --git a/worker/common/Cargo.toml b/worker/common/Cargo.toml index cb818b7a2d..701a3ae974 100644 --- a/worker/common/Cargo.toml +++ b/worker/common/Cargo.toml @@ -11,6 +11,11 @@ edition = "2024" log = { version = "0.4", default-features = false } codec = { workspace = true, default-features = false, features = ["derive"] } +scale-info = { workspace = true, default-features = false, features = [ + "derive", +] } +serde = { version = "1.0.104", optional = true, default-features = false } + ark-std = { workspace = true, default-features = false } polkavm-derive = { version = "0.32.0", optional = true } @@ -28,7 +33,9 @@ polkavm = ["polkavm-derive"] testing = [] std = [ + "serde", "codec/std", + "serde/std", "ark-std/std", "log/std", ] diff --git a/worker/common/src/error.rs b/worker/common/src/error.rs index 0f36a073c6..8c96d21018 100644 --- a/worker/common/src/error.rs +++ b/worker/common/src/error.rs @@ -1,9 +1,20 @@ -use codec::{Decode, Encode}; +use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; use crate::{WorkRequestId, WorkerSessionId}; /// Errors that can occur in the worker protocol implementation. -#[derive(Clone, Debug, Encode, Decode, PartialEq, Eq)] +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + TypeInfo, + DecodeWithMemTracking, + MaxEncodedLen +)] #[repr(u8)] pub enum ProtocolError { CustomProtocolError([u8; 3]) = 0, diff --git a/worker/common/src/lib.rs b/worker/common/src/lib.rs index e3055f0410..00b6cc7c03 100644 --- a/worker/common/src/lib.rs +++ b/worker/common/src/lib.rs @@ -1,7 +1,10 @@ #![cfg_attr(not(feature = "std"), no_std)] use ark_std::vec::Vec; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; pub mod error; pub use error::*; @@ -25,7 +28,7 @@ pub type BackendBitmask = u32; pub type ProtocolModuleConfigHash = [u8; 32]; /// Protocol initialization method. -#[derive(Clone, Debug, Encode, Decode)] +#[derive(Clone, Debug, Encode, Decode, TypeInfo)] pub enum ProtocolInitializationMethod { /// No initialization is needed, the module can be used directly after loading. NoInitializationNeeded, @@ -47,7 +50,7 @@ pub enum ResolvedInitializationMethod { } /// A protocol's module configuration, which contains the list of backends and their corresponding module definitions (code hash, context hash, etc). -#[derive(Clone, Debug, Encode, Decode)] +#[derive(Clone, Debug, Encode, Decode, TypeInfo)] pub struct ProtocolModuleConfig { /// The protocol id and version are included here to make sure the config hash includes the protocol information, which is used for caching the loaded module. pub protocol: Protocol, @@ -79,7 +82,17 @@ pub type BackendContextHash = [u8; 32]; pub type BackendModuleVersion = u32; /// For a give protocol, version and backend kind, the code hash and context hash of the module to be loaded. -#[derive(Clone, Debug, Encode, Decode)] +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + MaxEncodedLen, + TypeInfo, + DecodeWithMemTracking +)] pub struct BackendModuleDefinition { /// The module kind, used to determine which module code to load for the given backend kind. pub module_kind: BackendModuleKind, @@ -119,7 +132,20 @@ pub fn unpack_fat_results(fat_ptr: u64) -> Result<(u32, u32), ProtocolError> { /// The module code kind. /// /// Some backends like wasmer and wasmtime both support wasm modules and can share the same module code. -#[derive(Clone, Copy, Debug, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + Encode, + Decode, + PartialEq, + Eq, + PartialOrd, + Ord, + DecodeWithMemTracking, + MaxEncodedLen, + TypeInfo +)] #[repr(u8)] pub enum BackendModuleKind { Native = 0, @@ -131,7 +157,20 @@ pub enum BackendModuleKind { /// /// This is used to allow disabling certain backends in the future if they are found to be insecure or have other issues. /// It also allows us to have multiple backends for the same protocol if needed. -#[derive(Clone, Copy, Debug, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + Encode, + Decode, + PartialEq, + Eq, + PartialOrd, + Ord, + DecodeWithMemTracking, + MaxEncodedLen, + TypeInfo +)] #[repr(u32)] pub enum BackendKind { Native = 0, @@ -286,12 +325,28 @@ impl core::ops::BitOr for BackendBitmask { /// Protocol id. #[derive( - Clone, Copy, Debug, Default, Encode, Decode, PartialEq, Eq, PartialOrd, Ord + Clone, + Copy, + Debug, + Default, + Encode, + Decode, + PartialEq, + Eq, + PartialOrd, + Ord, + DecodeWithMemTracking, + MaxEncodedLen, + TypeInfo )] -pub struct ProtocolId(u16); +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ProtocolId(pub u16); pub type ProtocolNumber = u64; +/// The protocol id for the TESTING protocol. +pub const PROTOCOL_TESTING: ProtocolId = ProtocolId(0xFF); + /// The protocol id for the P-DART protocol. /// /// 0x50 is chosen as it is the ASCII code for 'P', which stands for Polymesh. @@ -299,8 +354,21 @@ pub const PROTOCOL_PDART: ProtocolId = ProtocolId(0x50); /// The protocol id and version. #[derive( - Clone, Copy, Debug, Default, Encode, Decode, PartialEq, Eq, PartialOrd, Ord + Clone, + Copy, + Debug, + Default, + Encode, + Decode, + PartialEq, + Eq, + PartialOrd, + Ord, + DecodeWithMemTracking, + MaxEncodedLen, + TypeInfo )] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Protocol { pub id: ProtocolId, pub version: ProtocolVersion, @@ -356,8 +424,21 @@ impl Protocol { /// The protocol version is used load the correct module for the given protocol and version. #[derive( - Clone, Copy, Debug, Default, Encode, Decode, PartialEq, Eq, PartialOrd, Ord + Clone, + Copy, + Debug, + Default, + Encode, + Decode, + PartialEq, + Eq, + PartialOrd, + Ord, + DecodeWithMemTracking, + MaxEncodedLen, + TypeInfo )] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct ProtocolVersion { pub major: u16, pub minor: u16, diff --git a/worker/extension/src/loader.rs b/worker/extension/src/loader.rs index 048e3fd44d..ea7f119460 100644 --- a/worker/extension/src/loader.rs +++ b/worker/extension/src/loader.rs @@ -15,10 +15,13 @@ use polymesh_worker_common::*; /// The storage items are: /// - `ProtocolConfigHash` (twox_128 = 7ada29e3bb7bec6691576e935738af7b): map Protocol => Hash /// - `ProtocolConfig` (twox_128 = 63d64a3f5c7590e883a2f77f74b5714d): map (Protocol, Hash) => ProtocolModuleConfig -/// - `ProtocolModuleCode` (twox_128 = deae62fbb378690611a6e5113d494b1b): map (Protocol, Hash) => Vec +/// - `ProtocolCode` (twox_128 = f2f6e3b671f605a1c3aa34c6bff56787): map (Protocol, Hash) => Vec /// - `ProtocolContext` (twox_128 = 27c0c49da7153d33d35d497fc4ac56f0): map (Protocol, Hash) => Vec /// /// The storage prefix is `concat(twox_128(pallet_name), twox_128(storage_name))`. +/// +/// Legacy (`module_version == 1`): +/// - `ProtocolModuleCode` (twox_128 = deae62fbb378690611a6e5113d494b1b): map (Protocol, Hash) => Vec (not SCALE encoded, just raw bytes, no zstd compression) /// Pallet WorkerModules storage prefix. pub const WORKER_MODULES_PREFIX: [u8; 16] = hex_literal::hex!("e67cf3f4b484981dea7be98c7cbbd979"); @@ -30,8 +33,11 @@ pub const PROTOCOL_CONFIG_HASH_PREFIX: [u8; 16] = /// The prefix for `ProtocolConfig` storage item, which double map (Protocol, Hash) => ProtocolModuleConfig. pub const PROTOCOL_CONFIG_PREFIX: [u8; 16] = hex_literal::hex!("63d64a3f5c7590e883a2f77f74b5714d"); -/// The prefix for `ProtocolModuleCode` storage item, which double map (Protocol, Hash) => Vec. -pub const PROTOCOL_MODULE_CODE_PREFIX: [u8; 16] = +/// The prefix for `ProtocolCode` storage item, which double map (Protocol, Hash) => Vec. +pub const PROTOCOL_CODE_PREFIX: [u8; 16] = hex_literal::hex!("f2f6e3b671f605a1c3aa34c6bff56787"); + +/// The legacy prefix for `ProtocolModuleCode` storage item, which double map (Protocol, Hash) => Vec. +pub const PROTOCOL_MODULE_CODE_LEGACY_PREFIX: [u8; 16] = hex_literal::hex!("deae62fbb378690611a6e5113d494b1b"); /// The prefix for `ProtocolContext` storage item, which double map (Protocol, Hash) => Vec. @@ -64,7 +70,16 @@ pub fn worker_modules_config_key( /// Generate the storage key for `WorkerModules::ProtocolModuleCode(protocol, code_hash)`. pub fn worker_modules_code_key(protocol: Protocol, code_hash: BackendCodeHash) -> Vec { - worker_modules_storage_key(PROTOCOL_MODULE_CODE_PREFIX, protocol, Some(code_hash)) + worker_modules_storage_key(PROTOCOL_CODE_PREFIX, protocol, Some(code_hash)) +} + +/// Generate the legacy storage key for `WorkerModules::ProtocolModuleCode(protocol, code_hash)`. +pub fn worker_modules_legacy_code_key(protocol: Protocol, code_hash: BackendCodeHash) -> Vec { + worker_modules_storage_key( + PROTOCOL_MODULE_CODE_LEGACY_PREFIX, + protocol, + Some(code_hash), + ) } /// Generate the storage key for `WorkerModules::ProtocolContext(protocol, context_hash)`. @@ -150,6 +165,18 @@ impl<'a> polymesh_worker::backend::BackendModuleLoader for SubstrateModuleLoader code_hash: BackendCodeHash, ) -> Option> { self.storage(&worker_modules_code_key(protocol, code_hash)) + // SCALE decode `Vec` + .and_then(|bytes| match Decode::decode(&mut &bytes[..]) { + Ok(decoded) => Some(decoded), + Err(e) => { + log::error!( + "Failed to decode protocol module code for protocol {:?}: {:?}", + protocol, + e + ); + None + } + }) // Verify the code hash matches the read code, to prevent malicious block producers. .and_then(|bytes| { verify_hash( @@ -169,6 +196,18 @@ impl<'a> polymesh_worker::backend::BackendModuleLoader for SubstrateModuleLoader ctx_hash: BackendContextHash, ) -> Option> { self.storage(&worker_modules_context_key(protocol, ctx_hash)) + // SCALE decode `Vec` + .and_then(|bytes| match Decode::decode(&mut &bytes[..]) { + Ok(decoded) => Some(decoded), + Err(e) => { + log::error!( + "Failed to decode protocol module context for protocol {:?}: {:?}", + protocol, + e + ); + None + } + }) // Verify the context hash matches the read context, to prevent malicious block producers. .and_then(|bytes| { verify_hash( diff --git a/worker/native/Cargo.toml b/worker/native/Cargo.toml index 203f01448d..f5d8624add 100644 --- a/worker/native/Cargo.toml +++ b/worker/native/Cargo.toml @@ -6,10 +6,12 @@ edition = "2024" [dependencies] log = "0.4" +codec = { workspace = true, default-features = false } sp-panic-handler = { workspace = true, default-features = false } # Protocols polymesh-worker-protocol-dart-v1 = { workspace = true, default-features = false, features = ["native"] } +polymesh-worker-protocol-testing = { workspace = true, default-features = false, features = ["native"] } # Worker interface polymesh-worker = { workspace = true, default-features = false } @@ -24,6 +26,7 @@ default = [ testing = [ "polymesh-worker-common/testing", "polymesh-worker-protocol-dart-v1/testing", + "polymesh-worker-protocol-testing/testing", ] debug_logging = [] @@ -38,7 +41,9 @@ parallel = [ std = [ "asm", "parallel", + "codec/std", "polymesh-worker/std", "polymesh-worker-common/std", "polymesh-worker-protocol-dart-v1/std", + "polymesh-worker-protocol-testing/std", ] diff --git a/worker/native/src/dart.rs b/worker/native/src/dart.rs new file mode 100644 index 0000000000..dcf9623494 --- /dev/null +++ b/worker/native/src/dart.rs @@ -0,0 +1,84 @@ +use polymesh_worker::backend::{BackendModule, BackendModuleInstance}; +use polymesh_worker_common::*; + +use polymesh_worker_protocol_dart_v1::*; + +#[derive(Clone, Debug)] +pub struct NativeDartModuleInstance; + +impl BackendModuleInstance for NativeDartModuleInstance { + fn allocate_scratch_pad(&mut self, _min_size: u32) -> Result<(u32, u32), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn release_scratch_pad(&mut self, _ptr: u32, _size: u32) -> Result<(), WorkerError> { + Ok(()) + } + + fn write_memory(&mut self, _ptr: u32, _data: &[u8]) -> Result<(), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn read_memory_into(&mut self, _ptr: u32, _buffer: &mut [u8]) -> Result<(), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn read_memory(&mut self, _ptr: u32, _len: u32) -> Result, WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn call_initialize(&mut self, _params_len: u32, _save: u32) -> Result { + Ok(0) + } + + fn call_execute(&mut self, _req_len: u32) -> Result { + Ok(0) + } + + fn initialize(&mut self, load_ctx: Option<&[u8]>) -> Result { + Ok(initialize(load_ctx).map_err(|_| WorkerError::ModuleInitializationFailed)? as u32) + } + + fn save_context(&mut self) -> Result>, WorkerError> { + Ok(Some( + save_context().map_err(|_| WorkerError::ModuleSaveContextFailed)?, + )) + } + + fn execute(&mut self, req: &WorkRequest) -> Result { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = sp_panic_handler::AbortGuard::force_unwind(); + execute_work_request(req) + })); + match result { + Ok(result) => Ok(result), + Err(panic) => { + let message = if let Some(message) = panic.downcast_ref::() { + format!( + "native worker panicked while executing a work request: {}", + message + ) + } else if let Some(message) = panic.downcast_ref::<&'static str>() { + format!( + "native worker panicked while executing a work request: {}", + message + ) + } else { + "native worker panicked while executing a work request".to_owned() + }; + log::warn!("NATIVE_WORKER: {}", message); + // The work request execution panicked, reject the work request (for proof validation this means the proof is rejected). + Ok(Err(ProtocolError::ExecuteWorkFailed)) + } + } + } +} + +#[derive(Clone, Debug)] +pub struct NativeDartModule; + +impl BackendModule for NativeDartModule { + fn instantiate(&self) -> Option> { + Some(Box::new(NativeDartModuleInstance)) + } +} diff --git a/worker/native/src/lib.rs b/worker/native/src/lib.rs index 7905c8997e..797cb47004 100644 --- a/worker/native/src/lib.rs +++ b/worker/native/src/lib.rs @@ -1,87 +1,13 @@ -use polymesh_worker::backend::{Backend, BackendModule, BackendModuleInstance}; -use polymesh_worker_common::*; - -use polymesh_worker_protocol_dart_v1::*; - -#[derive(Clone, Debug)] -pub struct NativeModuleInstance; - -impl BackendModuleInstance for NativeModuleInstance { - fn allocate_scratch_pad(&mut self, _min_size: u32) -> Result<(u32, u32), WorkerError> { - Err(WorkerError::ModuleMemoryError) - } - - fn release_scratch_pad(&mut self, _ptr: u32, _size: u32) -> Result<(), WorkerError> { - Ok(()) - } - - fn write_memory(&mut self, _ptr: u32, _data: &[u8]) -> Result<(), WorkerError> { - Err(WorkerError::ModuleMemoryError) - } - - fn read_memory_into(&mut self, _ptr: u32, _buffer: &mut [u8]) -> Result<(), WorkerError> { - Err(WorkerError::ModuleMemoryError) - } - - fn read_memory(&mut self, _ptr: u32, _len: u32) -> Result, WorkerError> { - Err(WorkerError::ModuleMemoryError) - } - - fn call_initialize(&mut self, _params_len: u32, _save: u32) -> Result { - Ok(0) - } - - fn call_execute(&mut self, _req_len: u32) -> Result { - Ok(0) - } +use codec::Decode; - fn initialize(&mut self, load_ctx: Option<&[u8]>) -> Result { - Ok(initialize(load_ctx).map_err(|_| WorkerError::ModuleInitializationFailed)? as u32) - } - - fn save_context(&mut self) -> Result>, WorkerError> { - Ok(Some( - save_context().map_err(|_| WorkerError::ModuleSaveContextFailed)?, - )) - } - - fn execute(&mut self, req: &WorkRequest) -> Result { - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _guard = sp_panic_handler::AbortGuard::force_unwind(); - execute_work_request(req) - })); - match result { - Ok(result) => Ok(result), - Err(panic) => { - let message = if let Some(message) = panic.downcast_ref::() { - format!( - "native worker panicked while executing a work request: {}", - message - ) - } else if let Some(message) = panic.downcast_ref::<&'static str>() { - format!( - "native worker panicked while executing a work request: {}", - message - ) - } else { - "native worker panicked while executing a work request".to_owned() - }; - log::warn!("NATIVE_WORKER: {}", message); - // The work request execution panicked, reject the work request (for proof validation this means the proof is rejected). - Ok(Err(ProtocolError::ExecuteWorkFailed)) - } - } - } -} +use polymesh_worker::backend::{Backend, BackendModule}; +use polymesh_worker_common::*; -#[derive(Clone, Debug)] -pub struct NativeModule; +mod dart; +mod testing; -impl BackendModule for NativeModule { - fn instantiate(&self) -> Option> { - Some(Box::new(NativeModuleInstance)) - } -} +use dart::NativeDartModule; +use testing::NativeTestingModule; #[derive(Clone, Debug)] pub struct NativeBackend; @@ -95,7 +21,23 @@ impl Backend for NativeBackend { BackendKind::Native } - fn load_module(&self, _module_bytes: &[u8]) -> Option> { - Some(Box::new(NativeModule)) + fn load_module(&self, module_bytes: &[u8]) -> Option> { + let protocol = Protocol::decode(&mut &module_bytes[..]).ok(); + match protocol { + Some(protocol) => { + // Native modules don't have module code bytes, so we just encode the protocol. + if protocol == polymesh_worker_protocol_dart_v1::PROTOCOL { + Some(Box::new(NativeDartModule)) + } else if protocol == polymesh_worker_protocol_testing::PROTOCOL { + Some(Box::new(NativeTestingModule)) + } else { + None + } + } + None => { + // Fallback to the native DART module. + Some(Box::new(NativeDartModule)) + } + } } } diff --git a/worker/native/src/testing.rs b/worker/native/src/testing.rs new file mode 100644 index 0000000000..ca890716e7 --- /dev/null +++ b/worker/native/src/testing.rs @@ -0,0 +1,84 @@ +use polymesh_worker::backend::{BackendModule, BackendModuleInstance}; +use polymesh_worker_common::*; + +use polymesh_worker_protocol_testing::*; + +#[derive(Clone, Debug)] +pub struct NativeTestingModuleInstance; + +impl BackendModuleInstance for NativeTestingModuleInstance { + fn allocate_scratch_pad(&mut self, _min_size: u32) -> Result<(u32, u32), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn release_scratch_pad(&mut self, _ptr: u32, _size: u32) -> Result<(), WorkerError> { + Ok(()) + } + + fn write_memory(&mut self, _ptr: u32, _data: &[u8]) -> Result<(), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn read_memory_into(&mut self, _ptr: u32, _buffer: &mut [u8]) -> Result<(), WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn read_memory(&mut self, _ptr: u32, _len: u32) -> Result, WorkerError> { + Err(WorkerError::ModuleMemoryError) + } + + fn call_initialize(&mut self, _params_len: u32, _save: u32) -> Result { + Ok(0) + } + + fn call_execute(&mut self, _req_len: u32) -> Result { + Ok(0) + } + + fn initialize(&mut self, load_ctx: Option<&[u8]>) -> Result { + Ok(initialize(load_ctx).map_err(|_| WorkerError::ModuleInitializationFailed)? as u32) + } + + fn save_context(&mut self) -> Result>, WorkerError> { + Ok(Some( + save_context().map_err(|_| WorkerError::ModuleSaveContextFailed)?, + )) + } + + fn execute(&mut self, req: &WorkRequest) -> Result { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = sp_panic_handler::AbortGuard::force_unwind(); + execute_work_request(req) + })); + match result { + Ok(result) => Ok(result), + Err(panic) => { + let message = if let Some(message) = panic.downcast_ref::() { + format!( + "native worker panicked while executing a work request: {}", + message + ) + } else if let Some(message) = panic.downcast_ref::<&'static str>() { + format!( + "native worker panicked while executing a work request: {}", + message + ) + } else { + "native worker panicked while executing a work request".to_owned() + }; + log::warn!("NATIVE_WORKER: {}", message); + // The work request execution panicked, reject the work request (for proof validation this means the proof is rejected). + Ok(Err(ProtocolError::ExecuteWorkFailed)) + } + } + } +} + +#[derive(Clone, Debug)] +pub struct NativeTestingModule; + +impl BackendModule for NativeTestingModule { + fn instantiate(&self) -> Option> { + Some(Box::new(NativeTestingModuleInstance)) + } +} diff --git a/worker/polymesh-worker-protocol-dart-v1.polkavm b/worker/polymesh-worker-protocol-dart-v1.polkavm index 8e09a04d3d..a210b14849 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.polkavm and b/worker/polymesh-worker-protocol-dart-v1.polkavm differ diff --git a/worker/polymesh-worker-protocol-dart-v1.polkavm.zst b/worker/polymesh-worker-protocol-dart-v1.polkavm.zst index 065d4518cb..b6717c8dae 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.polkavm.zst and b/worker/polymesh-worker-protocol-dart-v1.polkavm.zst differ diff --git a/worker/polymesh-worker-protocol-dart-v1.testing.polkavm b/worker/polymesh-worker-protocol-dart-v1.testing.polkavm index d8c3ade1c7..68081a86ba 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.testing.polkavm and b/worker/polymesh-worker-protocol-dart-v1.testing.polkavm differ diff --git a/worker/polymesh-worker-protocol-dart-v1.testing.polkavm.zst b/worker/polymesh-worker-protocol-dart-v1.testing.polkavm.zst index 0e40c27a0a..aaac5f6bf9 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.testing.polkavm.zst and b/worker/polymesh-worker-protocol-dart-v1.testing.polkavm.zst differ diff --git a/worker/polymesh-worker-protocol-dart-v1.testing.wasm b/worker/polymesh-worker-protocol-dart-v1.testing.wasm index 80a356ba7b..f0331fecca 100755 Binary files a/worker/polymesh-worker-protocol-dart-v1.testing.wasm and b/worker/polymesh-worker-protocol-dart-v1.testing.wasm differ diff --git a/worker/polymesh-worker-protocol-dart-v1.testing.wasm.zst b/worker/polymesh-worker-protocol-dart-v1.testing.wasm.zst index 2375d54577..9c981fadd9 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.testing.wasm.zst and b/worker/polymesh-worker-protocol-dart-v1.testing.wasm.zst differ diff --git a/worker/polymesh-worker-protocol-dart-v1.wasm b/worker/polymesh-worker-protocol-dart-v1.wasm index ded91fd549..a74f590c1f 100755 Binary files a/worker/polymesh-worker-protocol-dart-v1.wasm and b/worker/polymesh-worker-protocol-dart-v1.wasm differ diff --git a/worker/polymesh-worker-protocol-dart-v1.wasm.zst b/worker/polymesh-worker-protocol-dart-v1.wasm.zst index b9194567c5..184db66c13 100644 Binary files a/worker/polymesh-worker-protocol-dart-v1.wasm.zst and b/worker/polymesh-worker-protocol-dart-v1.wasm.zst differ diff --git a/worker/protocol/dart-v1/Cargo.toml b/worker/protocol/dart-v1/Cargo.toml index f4769c5107..f8f7ddaabf 100644 --- a/worker/protocol/dart-v1/Cargo.toml +++ b/worker/protocol/dart-v1/Cargo.toml @@ -7,10 +7,6 @@ repository = "https://github.com/PolymeshAssociation/Polymesh" description = "Polymesh DART Confidential Assets host functions" edition = "2024" -[lib] -path = "src/lib.rs" -crate-type = ["lib", "cdylib"] - [dependencies] log = "0.4" @@ -75,6 +71,14 @@ runtime = [ "bulletproofs/host_hash_to_curve", "polymesh-worker-extension", ] +runtime_only = [ + "native", + "impl_protocol", +] +use_host_curves = [ + "ark-ec/host_msm", + "bulletproofs/host_hash_to_curve", +] sp-io = [ "polymesh-dart/sp-io", diff --git a/worker/protocol/testing/Cargo.toml b/worker/protocol/testing/Cargo.toml new file mode 100644 index 0000000000..2494ef19b8 --- /dev/null +++ b/worker/protocol/testing/Cargo.toml @@ -0,0 +1,73 @@ +[package] +name = "polymesh-worker-protocol-testing" +version = "0.1.0" +authors = ["Polymesh Labs"] +license-file = "../../../LICENSE" +repository = "https://github.com/PolymeshAssociation/Polymesh" +description = "Worker module for testing Polymesh Worker" +edition = "2024" + +[dependencies] +log = "0.4" + +polymesh-worker-common = { workspace = true, default-features = false } +polymesh-worker-extension = { workspace = true, default-features = false, optional = true } + +# Substrate +scale-info = { workspace = true, default-features = false, features = [ + "derive", +] } +codec = { workspace = true, default-features = false, features = ["derive"] } +sp-std = { workspace = true, default-features = false } +bounded-collections = { workspace = true, default-features = false, features = ["scale-codec"] } + +[target.'cfg(target_family = "wasm")'.dependencies] +picoalloc = "5.2.0" + +[target.'cfg(target_env = "polkavm")'.dependencies] +polkavm-derive = { workspace = true, default-features = true } +picoalloc = "5.2.0" + +[features] +default = ["std", "testing", "version_v0"] + +testing = [] + +# Use feature flags to generate multiple versions of the testing module. +version_v0 = [] +version_v1 = [] +version_v2 = [] + +# Directly implement protocol logic (i.e. don't use a host function). +impl_protocol = [] + +polkavm = [ + "polymesh-worker-common/host_logger", + "polymesh-worker-common/polkavm", + "impl_protocol", +] + +wasm = [ + "polymesh-worker-common/host_logger", + "impl_protocol", +] + +native = [] + +runtime = [ + "native", + "polymesh-worker-extension", +] +runtime_only = [ + "native", + "impl_protocol", +] + +no_std = [] +std = [ + "impl_protocol", + "codec/std", + "sp-std/std", + "bounded-collections/std", +] +runtime-benchmarks = ["testing"] diff --git a/worker/protocol/testing/build_polkavm.sh b/worker/protocol/testing/build_polkavm.sh new file mode 100755 index 0000000000..9f6bce3776 --- /dev/null +++ b/worker/protocol/testing/build_polkavm.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +VERSION=${1:-"v0"} + +TARGET_JSON_PATH="$(polkatool get-target-json-path --bitness 64)" +#echo "$TARGET_JSON_PATH" + +crate="polymesh-worker-protocol-testing" +lib_name="polymesh_worker_protocol_testing" +elf_path="../../../target/riscv64emac-unknown-none-polkavm/release/$lib_name.elf" +output_path="$VERSION/$crate.polkavm" +rm -f "$output_path" "$elf_path" + +echo "> Building: '$crate' (-> $output_path)" + +RUSTFLAGS="--remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C codegen-units=1" \ +cargo rustc --crate-type cdylib \ + -Z build-std=core,alloc \ + --target $TARGET_JSON_PATH \ + --no-default-features \ + --features polkavm,version_$VERSION \ + --release --lib -p $crate + +polkatool link \ + --run-only-if-newer -s $elf_path \ + -o $output_path + +cargo run -r -p polymesh-worker-tools -- compress $output_path diff --git a/worker/protocol/testing/build_wasm.sh b/worker/protocol/testing/build_wasm.sh new file mode 100755 index 0000000000..c7f4ba732d --- /dev/null +++ b/worker/protocol/testing/build_wasm.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +VERSION=${1:-"v0"} + +target="wasm32-unknown-unknown" +crate="polymesh-worker-protocol-testing" +lib_name="polymesh_worker_protocol_testing" +wasm_path="../../../target/$target/release/$lib_name.wasm" + +output_path="$VERSION/$crate.wasm" +rm -f "$output_path" "$wasm_path" + +echo "> Building: '$crate' (-> $output_path)" + +#RUSTFLAGS="-C target-feature=+simd128,+wide-arithmetic --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ +RUSTFLAGS="-C target-feature=+simd128 --remap-path-prefix=$(pwd)= --remap-path-prefix=$HOME=~ -C strip=symbols -C codegen-units=1" \ + cargo rustc --crate-type cdylib \ + --target=$target \ + --no-default-features \ + --features wasm,version_$VERSION \ + --release --lib -p $crate + +cp $wasm_path $output_path + +cargo run -r -p polymesh-worker-tools -- compress $output_path diff --git a/worker/protocol/testing/rebuild.sh b/worker/protocol/testing/rebuild.sh new file mode 100755 index 0000000000..5894b04c14 --- /dev/null +++ b/worker/protocol/testing/rebuild.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -euo pipefail + +# v0.1.0 +./build_polkavm.sh && ./build_wasm.sh + +# v1.0.0 +./build_polkavm.sh v1 && ./build_wasm.sh v1 + +# v2.0.0 +./build_polkavm.sh v2 && ./build_wasm.sh v2 diff --git a/worker/protocol/testing/src/lib.rs b/worker/protocol/testing/src/lib.rs new file mode 100644 index 0000000000..01d1c73b87 --- /dev/null +++ b/worker/protocol/testing/src/lib.rs @@ -0,0 +1,497 @@ +// This file is part of the Polymesh distribution (https://github.com/PolymeshAssociation/Polymesh). +// Copyright (c) 2026 Polymesh Association + +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(feature = "native"), no_main)] + +#[cfg(feature = "polkavm")] +polkavm_derive::min_stack_size!(1); +#[cfg(feature = "polkavm")] +polkavm_derive::min_stack_size!(1024 * 1024); +#[cfg(feature = "polkavm")] +polkavm_derive::min_stack_size!(2); + +#[cfg(not(any(feature = "native", feature = "std")))] +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + #[cfg(target_family = "wasm")] + { + core::arch::wasm32::unreachable(); + } + + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + unsafe { + core::arch::asm!("unimp", options(noreturn)); + } +} + +#[cfg(not(feature = "native"))] +const HEAP_SIZE: usize = 10 * 1024 * 1024; + +#[cfg(not(feature = "native"))] +#[global_allocator] +static mut GLOBAL_ALLOC: picoalloc::Mutex< + picoalloc::Allocator>, +> = { + static mut ARRAY: picoalloc::Array<{ HEAP_SIZE }> = picoalloc::Array([0; HEAP_SIZE]); + + picoalloc::Mutex::new(picoalloc::Allocator::new(unsafe { + picoalloc::ArrayPointer::new(&raw mut ARRAY) + })) +}; + +/// The size of the scratch pad used to hold temporary data to be passed between the host and the module. +#[cfg(not(feature = "native"))] +const SCRATCH_SIZE: usize = 4 * 1024 * 1024; + +#[cfg(not(feature = "native"))] +static mut SCRATCH: picoalloc::Array<{ SCRATCH_SIZE }> = picoalloc::Array([0; SCRATCH_SIZE]); + +#[cfg(not(feature = "native"))] +#[allow(static_mut_refs)] +pub fn mut_scratch() -> &'static mut [u8] { + unsafe { &mut SCRATCH.0 } +} + +#[cfg(not(feature = "native"))] +#[allow(static_mut_refs)] +pub fn scratch() -> &'static [u8] { + unsafe { &SCRATCH.0 } +} + +#[cfg(not(feature = "native"))] +use polymesh_worker_common::pack_fat_pointer; +#[cfg(not(feature = "impl_protocol"))] +use polymesh_worker_common::{ + WORK_CONFIG_FLAG_USE_CACHE, WorkRequestId, WorkerError, unpack_work_status_flags_and_id, +}; + +use polymesh_worker_common::{ + PROTOCOL_TESTING, Protocol, ProtocolError, ProtocolVersion, WorkRequest, WorkResponse, + WorkResponseResult, WorkerSessionId, +}; + +#[cfg(not(feature = "impl_protocol"))] +use polymesh_worker_extension::*; + +use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +use sp_std::{vec, vec::Vec}; + +const fn protocol_version() -> ProtocolVersion { + #[cfg(feature = "version_v1")] + { + ProtocolVersion::new(1, 0, 0) + } + #[cfg(feature = "version_v2")] + { + ProtocolVersion::new(2, 0, 0) + } + #[cfg(not(any(feature = "version_v1", feature = "version_v2")))] + { + ProtocolVersion::new(0, 1, 0) + } +} + +pub const PROTOCOL: Protocol = Protocol { + id: PROTOCOL_TESTING, + version: protocol_version(), +}; + +/// Verify version request. +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + TypeInfo, + DecodeWithMemTracking, + MaxEncodedLen +)] +pub struct VerifyVersionRequest { + pub protocol: Protocol, +} + +impl VerifyVersionRequest { + pub fn do_verify(&self) -> Result { + if self.protocol != PROTOCOL { + return Err(Error::VerifyFailed.into()); + } + Ok(VerifyVersionResponse {}) + } +} + +/// Test work request. +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + TypeInfo, + DecodeWithMemTracking, + MaxEncodedLen +)] +pub enum TestWorkRequest { + VerifyVersion(VerifyVersionRequest), +} + +impl TestWorkRequest { + fn do_execute(self) -> Result { + match self { + Self::VerifyVersion(req) => { + let res = req.do_verify()?; + Ok(TestWorkResponse::VerifyVersion(res)) + } + } + } + + pub fn execute_work(req: &WorkRequest) -> Result { + let req: Self = req.decode()?; + req.do_execute() + } +} + +#[cfg(feature = "impl_protocol")] +impl TestWorkRequest { + pub fn execute(self) -> Result { + self.do_execute() + } + + pub fn session_execute_and_wait( + self, + _session_id: WorkerSessionId, + ) -> Result { + self.do_execute() + } +} + +#[cfg(not(feature = "impl_protocol"))] +impl TestWorkRequest { + /// Execute a work request without a session, and return the results. + pub fn execute(self) -> Result { + let req = WorkRequest::new(&self); + let backends = BackendKind::all_mask(); + + match native_polymesh_worker::execute_request(PROTOCOL.to_number(), backends, req.0) { + Ok(Ok(resp)) => Ok(resp.decode()?), + Ok(Err(err)) => { + // This is a protocol error (i.e. invalid proof). + Err(err) + } + Err(err) => { + // Fallback to runtime execution if the host execution fails, to allow older nodes to continue syncing. + log::debug!( + "Host failed to execute work, falling back to runtime execution: {:?}", + err + ); + self.do_execute() + } + } + } + + /// Execute the work request in the given session, and don't wait for the results. The results can be retrieved later using `session_get_result` with the returned request id. + pub fn session_execute( + self, + session_id: WorkerSessionId, + ) -> Result { + let req = WorkRequest::new(&self); + + let status_flag_and_id = native_polymesh_worker::session_execute_request( + session_id, + WORK_CONFIG_FLAG_USE_CACHE, + req.0, + ); + let (status, _flags, request_id) = unpack_work_status_flags_and_id(status_flag_and_id); + + match status { + WorkStatus::Pending | WorkStatus::Completed => Ok(request_id), + WorkStatus::ExecutionFailedFallbackToRuntime => { + // Fallback to runtime execution if the host execution fails, to allow older nodes to continue syncing. + log::debug!( + "Host failed to execute work, falling back to runtime execution for session id: {}, request id: {}", + session_id, + request_id + ); + let result = self.do_execute().map(WorkResponse::new); + + // Push the results back to the session, and return the request id if the push is successful. + WorkerError::result_from_u64(native_polymesh_worker::session_push_result( + session_id, request_id, result, + )) + .map_err(|err| { + log::error!( + "Failed to push result for session id: {}, request id: {}, error: {:?}", + session_id, + request_id, + err + ); + ProtocolError::ExecuteWorkFailed + })?; + + Ok(request_id) + } + WorkStatus::SessionNotFound => Err(ProtocolError::ExecuteWorkFailed), + WorkStatus::Unknown => Err(ProtocolError::UnexpectedResponse), + } + } + + /// Execute the work request in the given session and wait for the results. + pub fn session_execute_and_wait( + self, + session_id: WorkerSessionId, + ) -> Result { + let req = WorkRequest::new(&self); + + match native_polymesh_worker::session_execute_request_and_wait( + session_id, + WORK_CONFIG_FLAG_USE_CACHE, + req.0, + ) { + Ok(Ok(resp)) => Ok(resp.decode()?), + Ok(Err(err)) => { + // This is a protocol error (i.e. invalid proof). + Err(err) + } + Err(err) => { + // Fallback to runtime execution if the host execution fails, to allow older nodes to continue syncing. + log::debug!( + "Host failed to execute work, falling back to runtime execution: {:?}", + err + ); + self.do_execute() + } + } + } +} + +/// Verify version response. +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + TypeInfo, + DecodeWithMemTracking, + MaxEncodedLen +)] +pub struct VerifyVersionResponse {} + +/// Test work response. +#[derive( + Clone, + Debug, + Encode, + Decode, + PartialEq, + Eq, + TypeInfo, + DecodeWithMemTracking, + MaxEncodedLen +)] +pub enum TestWorkResponse { + VerifyVersion(VerifyVersionResponse), +} + +#[cfg(not(feature = "impl_protocol"))] +impl TestWorkResponse { + /// Get the work response results from the session and decode it into the expected response type. + pub fn session_get_result( + session_id: WorkerSessionId, + request_id: WorkRequestId, + ) -> Result { + match native_polymesh_worker::session_get_result(session_id, request_id) { + Ok(Ok(resp)) => Ok(resp.decode()?), + Ok(Err(err)) => { + // This is a protocol error (i.e. invalid proof). + Err(err) + } + Err(err) => { + // Fallback to runtime execution if the host execution fails, to allow older nodes to continue syncing. + log::debug!( + "Host failed to get result for session id: {}, request id: {}, falling back to runtime execution: {:?}", + session_id, + request_id, + err + ); + Err(ProtocolError::UnexpectedResponse) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Error { + VerifyFailed, + InvalidParams, + UnexpectedResponse, +} + +impl From for ProtocolError { + fn from(err: Error) -> Self { + let err_u8 = err as u8; + ProtocolError::custom_error([err_u8, 0, 0]) + } +} + +pub fn execute_work_request(req: &WorkRequest) -> WorkResponseResult { + let res = TestWorkRequest::execute_work(req)?; + Ok(WorkResponse::new(res)) +} + +/// Get the scratch pad pointer. +#[cfg(not(feature = "native"))] +#[cfg_attr(feature = "polkavm", polkavm_derive::polkavm_export)] +#[unsafe(no_mangle)] +pub extern "C" fn get_scratch_pad() -> u32 { + scratch().as_ptr() as u32 +} + +/// Get the scratch pad size. +#[cfg(not(feature = "native"))] +#[cfg_attr(feature = "polkavm", polkavm_derive::polkavm_export)] +#[unsafe(no_mangle)] +pub extern "C" fn get_scratch_pad_size() -> u32 { + scratch().len() as u32 +} + +/// Dummy parameters. +static mut DUMMY_PARAMS: Option> = None; + +const PARAMS_SIZE: usize = 10 * 1024; + +fn init_params() -> Result { + // Initialize the parameters with dummy data. + let params = vec![0u8; PARAMS_SIZE]; + unsafe { + DUMMY_PARAMS = Some(params); + } + Ok(PARAMS_SIZE) +} + +fn load_params(params: &[u8]) -> Result<(), Error> { + // check that the params length is correct + if params.len() != PARAMS_SIZE { + return Err(Error::InvalidParams); + } + // Save the params to the dummy parameters. + unsafe { + DUMMY_PARAMS = Some(params.to_vec()); + } + Ok(()) +} + +fn save_params(params: &mut Vec) -> Result { + // Save the dummy parameters to the given params vector. + #[allow(static_mut_refs)] + unsafe { + if let Some(dummy_params) = &DUMMY_PARAMS { + params.clear(); + params.extend_from_slice(dummy_params); + Ok(dummy_params.len()) + } else { + Err(Error::InvalidParams) + } + } +} + +#[cfg(feature = "native")] +pub fn initialize(load_ctx: Option<&[u8]>) -> Result { + if let Some(ctx) = load_ctx { + load_params(ctx)?; + Ok(ctx.len() as u32) + } else { + Ok(init_params()? as u32) + } +} + +#[cfg(feature = "native")] +pub fn save_context() -> Result, Error> { + let mut params_bytes = Vec::new(); + save_params(&mut params_bytes)?; + Ok(params_bytes) +} + +/// Initialize the module with the given parameters. +/// +/// If `params_len` is 0, the module will initialize the parameters and return the length of the initialized parameters. +/// If `params_len` is greater than 0, the module will load the parameters from the scratch pad and return the length of the loaded parameters. +/// If `save` is true, the module will save the initialized parameters to the scratch pad before returning. +#[cfg(not(feature = "native"))] +#[cfg_attr(feature = "polkavm", polkavm_derive::polkavm_export)] +#[unsafe(no_mangle)] +pub extern "C" fn initialize(params_len: u32, save: u32) -> u64 { + // Try to initialize the host logger. Only the first call to `initialize` will be able to initialize the logger. + let _ = polymesh_worker_common::logger::init(); + + let params_len = params_len; + if params_len > 0 { + let params_bytes = &scratch()[..params_len as usize]; + if load_params(params_bytes).is_ok() { + // Only return the length of the parameters to indicate success. + return pack_fat_pointer(0, params_len as u32) as u64; + } + } else { + if let Ok(len) = init_params() { + if save != 0 { + let mut params_bytes = unsafe { + let scratch = mut_scratch(); + Vec::from_raw_parts(scratch.as_mut_ptr(), 0, scratch.len()) + }; + let len = if let Ok(len) = save_params(&mut params_bytes) { + len as u32 + } else { + 0 + }; + // Prevent the Vec from deallocating the scratch memory. + core::mem::forget(params_bytes); + // Return the pointer and length of the saved parameters as a fat pointer. + return pack_fat_pointer(scratch().as_ptr() as u32, len as u32) as u64; + } + // Only return the length of the parameters to indicate success. + return pack_fat_pointer(0, len as u32) as u64; + } + } + 0 +} + +#[cfg(not(feature = "native"))] +#[cfg_attr(feature = "polkavm", polkavm_derive::polkavm_export)] +#[unsafe(no_mangle)] +pub extern "C" fn execute(req_len: u32) -> u64 { + let req_bytes = &scratch()[..req_len as usize]; + let req: WorkRequest = match Decode::decode(&mut &req_bytes[..]) { + Ok(req) => req, + Err(_) => return 0, + }; + + // Execute the request and get the response. + match execute_work_request(&req) { + Ok(res) => { + // Write the response to the scratch buffer and return a fat pointer to it. + let res_bytes = res.0; + let res_len = res_bytes.len(); + let scratch = mut_scratch(); + if scratch.len() < res_len { + // The response is too large to fit in the scratch buffer, return an error. + let err = ProtocolError::UnexpectedResponse.to_u32(); + return pack_fat_pointer(err, u32::MAX) as u64; + } + scratch[..res_len].copy_from_slice(&res_bytes); + + pack_fat_pointer(scratch.as_ptr() as u32, res_len as u32) as u64 + } + Err(err) => { + let err_u32 = err.to_u32(); + + // Return the error code as a fat pointer with `len` set to `u32::MAX` to indicate an error. + pack_fat_pointer(err_u32, u32::MAX) as u64 + } + } +} diff --git a/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm new file mode 100644 index 0000000000..c2c0cb7575 Binary files /dev/null and b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm differ diff --git a/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm.zst b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm.zst new file mode 100644 index 0000000000..eeddf749e7 Binary files /dev/null and b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.polkavm.zst differ diff --git a/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm new file mode 100755 index 0000000000..4700df1fb5 Binary files /dev/null and b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm differ diff --git a/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm.zst b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm.zst new file mode 100644 index 0000000000..b0d695d4c0 Binary files /dev/null and b/worker/protocol/testing/v0/polymesh-worker-protocol-testing.wasm.zst differ diff --git a/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm new file mode 100644 index 0000000000..f160221034 Binary files /dev/null and b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm differ diff --git a/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm.zst b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm.zst new file mode 100644 index 0000000000..49b502afab Binary files /dev/null and b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.polkavm.zst differ diff --git a/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm new file mode 100755 index 0000000000..e93e22867a Binary files /dev/null and b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm differ diff --git a/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm.zst b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm.zst new file mode 100644 index 0000000000..e581b40f8e Binary files /dev/null and b/worker/protocol/testing/v1/polymesh-worker-protocol-testing.wasm.zst differ diff --git a/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm new file mode 100644 index 0000000000..d8a753c3b7 Binary files /dev/null and b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm differ diff --git a/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm.zst b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm.zst new file mode 100644 index 0000000000..06664cb455 Binary files /dev/null and b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.polkavm.zst differ diff --git a/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm new file mode 100755 index 0000000000..2e24785cea Binary files /dev/null and b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm differ diff --git a/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm.zst b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm.zst new file mode 100644 index 0000000000..e5f3db4922 Binary files /dev/null and b/worker/protocol/testing/v2/polymesh-worker-protocol-testing.wasm.zst differ diff --git a/worker/src/backend.rs b/worker/src/backend.rs index 43e718014e..a909f04367 100644 --- a/worker/src/backend.rs +++ b/worker/src/backend.rs @@ -5,6 +5,7 @@ use codec::Encode; use polymesh_worker_common::*; use crate::cache::modules::{BackendModuleCache, ProtocolModuleRef}; +use crate::decompress_module_code; #[cfg(feature = "polkavm")] mod polkavm; @@ -102,6 +103,8 @@ impl Backends { for backend in &self.backends { let kind = backend.kind(); if let Some(module_bytes) = loader.get_module_bytes(protocol, kind) { + // Decompress module bytes if it was compressed. + let module_bytes = decompress_module_code(&module_bytes)?; if let Some(module) = backend.load_module(&module_bytes) { return Some(module); } @@ -338,6 +341,10 @@ pub trait BackendModule: Send + Sync { } /// A trait for backends that can be used to verify proofs. +/// +/// Supported versions: +/// - Legancy (version 1) (Polymesh v8.0 - v8.1.1) - Code and Context was stored as raw bytes (not SCALE-encode `Vec`). +/// - Current (version 2) (Polymesh v8.2 and later) - Code and Context are stored as SCALE-encoded `Vec`. pub trait Backend: Send + Sync { fn new_boxed() -> Result, WorkerError> where @@ -348,12 +355,12 @@ pub trait Backend: Send + Sync { /// Maximum supported module version, used for compatibility checking. fn max_supported_version(&self) -> BackendModuleVersion { - 1 + 2 } /// Minimum supported module version, used for compatibility checking. fn min_supported_version(&self) -> BackendModuleVersion { - 1 + 2 } /// Check if a module is compatible. diff --git a/worker/src/cache/modules.rs b/worker/src/cache/modules.rs index 191e25d1b6..6d0f65b2c9 100644 --- a/worker/src/cache/modules.rs +++ b/worker/src/cache/modules.rs @@ -6,6 +6,7 @@ use polymesh_worker_common::*; use crate::{ StaticModules, backend::{BACKENDS, Backend, BackendModule, BackendModuleInstance, BackendModuleLoader}, + decompress_module_code, }; pub const MAX_MODULE_INSTANCES_PER_PROTOCOL: usize = 32; @@ -102,6 +103,9 @@ impl ProtocolModule { let module_bytes = loader.get_module_code_bytes(protocol, module_kind, module_def.code_hash)?; + // Decompress module bytes if it was compressed. + let module_bytes = decompress_module_code(&module_bytes)?; + // Try loading the module into the backend. If this fails, it means that the module code is not compatible with this backend. let module = backend.load_module(&module_bytes)?; diff --git a/worker/src/lib.rs b/worker/src/lib.rs index 2895e9b8dc..3f9e7ab887 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use codec::Encode; pub use polymesh_worker_common::{ @@ -8,7 +10,8 @@ pub use polymesh_worker_common::{ WorkerVersion, config::*, error::*, }; use polymesh_worker_common::{ - BackendModuleKind, ProtocolInitializationMethod, ProtocolModuleConfig, ProtocolModuleConfigHash, + BackendModuleKind, PROTOCOL_TESTING, ProtocolInitializationMethod, ProtocolModuleConfig, + ProtocolModuleConfigHash, }; pub mod backend; @@ -39,36 +42,94 @@ pub fn decompress_module_code(bytes: &[u8]) -> Option> { } } -pub struct StaticModules { - initialized: bool, +pub struct StaticProtocol { config: ProtocolModuleConfig, config_hash: ProtocolModuleConfigHash, + polkavm_code: &'static [u8], polkavm_code_hash: BackendCodeHash, + wasm_code: &'static [u8], wasm_code_hash: BackendCodeHash, + native_code: Vec, native_code_hash: BackendCodeHash, } -impl StaticModules { - pub fn new() -> Self { +impl StaticProtocol { + pub fn new(protocol: Protocol, polkavm_code: &'static [u8], wasm_code: &'static [u8]) -> Self { + let mut modules = Vec::new(); + + let polkavm_code_hash = blake2b256_hash(polkavm_code); + let wasm_code_hash = blake2b256_hash(wasm_code); + let native_code = protocol.encode(); + let native_code_hash = blake2b256_hash(&native_code); + // Setup module definitions for the protocol config. + modules.push(BackendModuleDefinition { + module_kind: BackendModuleKind::PolkaVM, + module_version: 2, + code_hash: polkavm_code_hash, + }); + modules.push(BackendModuleDefinition { + module_kind: BackendModuleKind::Wasm, + module_version: 2, + code_hash: wasm_code_hash, + }); + modules.push(BackendModuleDefinition { + module_kind: BackendModuleKind::Native, + module_version: 2, + code_hash: native_code_hash, + }); let config = ProtocolModuleConfig { - protocol: Protocol { - id: PROTOCOL_PDART, - version: ProtocolVersion::new(0, 1, 0), - }, + protocol, initialization_method: ProtocolInitializationMethod::SaveContextFromFirstInstance, - modules: Vec::new(), + modules, }; + let config_hash = blake2b256_hash(&config.encode()); + Self { - initialized: false, config, - config_hash: [0u8; 32], - polkavm_code_hash: [0u8; 32], - wasm_code_hash: [0u8; 32], - native_code_hash: [42u8; 32], + config_hash, + polkavm_code, + polkavm_code_hash, + wasm_code, + wasm_code_hash, + native_code, + native_code_hash, } } - fn polkavm_bytes(&self) -> &'static [u8] { + pub fn get_code_bytes( + &self, + kind: BackendModuleKind, + code_hash: BackendCodeHash, + ) -> Option> { + match kind { + BackendModuleKind::PolkaVM if code_hash == self.polkavm_code_hash => { + Some(self.polkavm_code.to_vec()) + } + BackendModuleKind::Wasm if code_hash == self.wasm_code_hash => { + Some(self.wasm_code.to_vec()) + } + BackendModuleKind::Native if code_hash == self.native_code_hash => { + Some(self.native_code.clone()) + } + _ => None, + } + } +} + +pub struct StaticModules { + initialized: bool, + protocols: BTreeMap, +} + +impl StaticModules { + pub fn new() -> Self { + Self { + initialized: false, + protocols: BTreeMap::new(), + } + } + + fn dart_polkavm_bytes(&self) -> &'static [u8] { #[cfg(not(feature = "testing"))] { include_bytes!("../polymesh-worker-protocol-dart-v1.polkavm.zst") @@ -79,7 +140,7 @@ impl StaticModules { } } - fn wasm_bytes(&self) -> &'static [u8] { + fn dart_wasm_bytes(&self) -> &'static [u8] { #[cfg(not(feature = "testing"))] { include_bytes!("../polymesh-worker-protocol-dart-v1.wasm.zst") @@ -94,27 +155,34 @@ impl StaticModules { if self.initialized { return; } - // Precompute the code and context hashes for the static modules. - self.polkavm_code_hash = blake2b256_hash(self.polkavm_bytes()); - self.wasm_code_hash = blake2b256_hash(self.wasm_bytes()); + // Add P-DART protocol static modules. + let protocol = Protocol { + id: PROTOCOL_PDART, + version: ProtocolVersion::new(0, 1, 0), + }; + self.protocols.insert( + protocol, + StaticProtocol::new(protocol, self.dart_polkavm_bytes(), self.dart_wasm_bytes()), + ); + + // Add Testing protocol static modules if the testing feature is enabled. + //#[cfg(feature = "testing")] + { + let protocol = Protocol { + id: PROTOCOL_TESTING, + version: ProtocolVersion::new(0, 1, 0), + }; + let polkavm_code = include_bytes!( + "../protocol/testing/v0/polymesh-worker-protocol-testing.polkavm.zst" + ); + let wasm_code = + include_bytes!("../protocol/testing/v0/polymesh-worker-protocol-testing.wasm.zst"); + self.protocols.insert( + protocol, + StaticProtocol::new(protocol, polkavm_code, wasm_code), + ); + } - // Setup module definitions for the protocol config. - self.config.modules.push(BackendModuleDefinition { - module_kind: BackendModuleKind::PolkaVM, - module_version: 1, - code_hash: self.polkavm_code_hash, - }); - self.config.modules.push(BackendModuleDefinition { - module_kind: BackendModuleKind::Wasm, - module_version: 1, - code_hash: self.wasm_code_hash, - }); - self.config.modules.push(BackendModuleDefinition { - module_kind: BackendModuleKind::Native, - module_version: 1, - code_hash: self.native_code_hash, - }); - self.config_hash = blake2b256_hash(&self.config.encode()); self.initialized = true; } } @@ -124,9 +192,9 @@ impl backend::BackendModuleLoader for StaticModules { &mut self, protocol: Protocol, ) -> Option { - if protocol.id == PROTOCOL_PDART { - self.initialize(); - Some(self.config_hash) + self.initialize(); + if let Some(static_protocol) = self.protocols.get(&protocol) { + Some(static_protocol.config_hash) } else { None } @@ -138,9 +206,13 @@ impl backend::BackendModuleLoader for StaticModules { protocol: Protocol, config_hash: ProtocolModuleConfigHash, ) -> Option { - if protocol.id == PROTOCOL_PDART && config_hash == self.config_hash { - self.initialize(); - Some(self.config.clone()) + self.initialize(); + if let Some(static_protocol) = self.protocols.get(&protocol) { + if static_protocol.config_hash == config_hash { + Some(static_protocol.config.clone()) + } else { + None + } } else { None } @@ -152,18 +224,9 @@ impl backend::BackendModuleLoader for StaticModules { kind: BackendModuleKind, code_hash: BackendCodeHash, ) -> Option> { - if protocol.id == PROTOCOL_PDART { - self.initialize(); - match kind { - BackendModuleKind::PolkaVM if code_hash == self.polkavm_code_hash => { - decompress_module_code(self.polkavm_bytes()) - } - BackendModuleKind::Wasm if code_hash == self.wasm_code_hash => { - decompress_module_code(self.wasm_bytes()) - } - BackendModuleKind::Native if code_hash == self.native_code_hash => Some(vec![]), - _ => None, - } + self.initialize(); + if let Some(static_protocol) = self.protocols.get(&protocol) { + static_protocol.get_code_bytes(kind, code_hash) } else { None } diff --git a/worker/src/worker.rs b/worker/src/worker.rs index 1fd83483ca..bfc5735602 100644 --- a/worker/src/worker.rs +++ b/worker/src/worker.rs @@ -576,10 +576,15 @@ impl PolymeshWorker { let module = self .backend .load_protocol(config.backends, protocol, loader); + let loaded = module.is_some(); let mut inner = self.inner.write(); let session = inner.create_session(config, protocol, module); - log::debug!("Started session with id: {}", session.id); + log::debug!( + "Started session with id: {}, module loaded: {}", + session.id, + loaded + ); session }