refactor(spider-core)!: Unify external resource group credentials (fixes #468).

components/spider-client/src/client.rs

@@ -9,6 +9,7 @@ use spider_core::types::id::JobId;
 use spider_core::types::id::ResourceGroupId;
 use spider_core::types::io::TaskInput;
 use spider_core::types::io::TaskOutput;
+use spider_core::types::resource_group::ExternalResourceGroupCredentials;
 use spider_utils::grpc::retry::RetryConfig;
 use tonic::transport::Endpoint;
 
@@ -181,12 +182,9 @@ impl SpiderClient {
     /// * [`ClientError::Server`] for any other server-reported error.
     pub async fn add_resource_group(
         &self,
-        external_resource_group_id: String,
-        password: Vec<u8>,
+        credentials: ExternalResourceGroupCredentials,
     ) -> Result<ResourceGroupId, ClientError> {
-        self.resource_group
-            .add_resource_group(external_resource_group_id, password)
-            .await
+        self.resource_group.add_resource_group(credentials).await
     }
 
     /// Verifies a resource group's password.
@@ -308,6 +306,7 @@ fn assert_client_futures_send(
     resource_group_id: ResourceGroupId,
     job_id: JobId,
     task_graph: &TaskGraph,
+    credentials: ExternalResourceGroupCredentials,
 ) {
     const fn assert_send<FutureType: Send>(_: &FutureType) {}
     assert_send(&client.submit_job(resource_group_id, task_graph, Vec::new()));
@@ -316,6 +315,6 @@ fn assert_client_futures_send(
     assert_send(&client.get_job_state(job_id));
     assert_send(&client.get_job_outputs(job_id));
     assert_send(&client.get_job_error(job_id));
-    assert_send(&client.add_resource_group(String::new(), Vec::new()));
+    assert_send(&client.add_resource_group(credentials));
     assert_send(&client.verify_resource_group(resource_group_id, Vec::new()));
 }

components/spider-client/src/grpc/resource_group.rs

@@ -3,6 +3,7 @@
 use std::num::NonZeroUsize;
 
 use spider_core::types::id::ResourceGroupId;
+use spider_core::types::resource_group::ExternalResourceGroupCredentials;
 use spider_proto_rust::storage::ResourceGroupManagementServiceClient;
 use spider_proto_rust::storage::{self};
 use spider_utils::grpc::client::ConnectionPool;
@@ -66,15 +67,13 @@ impl ResourceGroupManagementClient {
     /// * Forwards [`ResourceGroupManagementServiceClient::add_resource_group`]'s status on failure.
     pub async fn add_resource_group(
         &self,
-        external_resource_group_id: String,
-        password: Vec<u8>,
+        credentials: ExternalResourceGroupCredentials,
     ) -> Result<ResourceGroupId, ClientError> {
         let pool = self.connection_pool.clone();
         let response = call_with_retry(self.retry_config, move || {
             let mut client = pool.get_client();
             let request = storage::AddResourceGroupRequest {
-                external_resource_group_id: external_resource_group_id.clone(),
-                password: password.clone(),
+                credentials: Some((&credentials).into()),
             };
             async move { client.add_resource_group(request).await }
         })

components/spider-core/Cargo.toml

@@ -11,6 +11,7 @@ path = "src/lib.rs"
 non-empty-string = { workspace = true }
 rand = { workspace = true }
 rmp-serde = { workspace = true }
+secrecy = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

components/spider-core/src/types/mod.rs

@@ -1,3 +1,4 @@
 pub mod id;
 pub mod io;
+pub mod resource_group;
 pub mod scheduler;

components/spider-core/src/types/resource_group.rs

@@ -0,0 +1,87 @@
+//! Resource group types shared across Spider components.
+
+use secrecy::ExposeSecret;
+use secrecy::SecretSlice;
+
+/// Environment variable that supplies the external resource group ID.
+pub const EXTERNAL_RESOURCE_GROUP_ID_ENV: &str = "SPIDER_EXTERNAL_RESOURCE_GROUP_ID";
+
+/// Environment variable that supplies the external resource group password.
+pub const EXTERNAL_RESOURCE_GROUP_PASSWORD_ENV: &str = "SPIDER_EXTERNAL_RESOURCE_GROUP_PASSWORD";
+
+/// Credentials identifying and authenticating an external resource group.
+#[derive(Debug, Clone)]
+pub struct ExternalResourceGroupCredentials {
+    /// The external resource group ID.
+    external_resource_group_id: String,
+
+    /// The resource group password.
+    password: SecretSlice<u8>,
+}
+
+impl ExternalResourceGroupCredentials {
+    /// Factory function.
+    ///
+    /// # Returns
+    ///
+    /// The newly created external resource group credentials.
+    #[must_use]
+    pub fn new(external_resource_group_id: String, password: Vec<u8>) -> Self {
+        Self {
+            external_resource_group_id,
+            password: password.into(),
+        }
+    }
+
+    /// Factory function.
+    ///
+    /// Reads the external resource group credentials from the [`EXTERNAL_RESOURCE_GROUP_ID_ENV`]
+    /// and [`EXTERNAL_RESOURCE_GROUP_PASSWORD_ENV`] environment variables.
+    ///
+    /// # Returns
+    ///
+    /// The credentials on success.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    ///
+    /// * [`ExternalResourceGroupCredentialsError::MissingEnvVar`] if either environment variable is
+    ///   unset.
+    pub fn from_env() -> Result<Self, ExternalResourceGroupCredentialsError> {
+        let external_resource_group_id =
+            std::env::var(EXTERNAL_RESOURCE_GROUP_ID_ENV).map_err(|_| {
+                ExternalResourceGroupCredentialsError::MissingEnvVar(EXTERNAL_RESOURCE_GROUP_ID_ENV)
+            })?;
+        let password = std::env::var(EXTERNAL_RESOURCE_GROUP_PASSWORD_ENV).map_err(|_| {
+            ExternalResourceGroupCredentialsError::MissingEnvVar(
+                EXTERNAL_RESOURCE_GROUP_PASSWORD_ENV,
+            )
+        })?;
+        Ok(Self::new(external_resource_group_id, password.into_bytes()))
+    }
+
+    /// # Returns
+    ///
+    /// The external resource group ID.
+    #[must_use]
+    pub fn get_external_resource_group_id(&self) -> &str {
+        &self.external_resource_group_id
+    }
+
+    /// # Returns
+    ///
+    /// The resource group password.
+    #[must_use]
+    pub fn get_password(&self) -> &[u8] {
+        self.password.expose_secret()
+    }
+}
+
+/// An error returned while reading external resource group credentials from the environment.
+#[derive(Debug, thiserror::Error)]
+pub enum ExternalResourceGroupCredentialsError {
+    /// A required environment variable is unavailable.
+    #[error("required environment variable `{0}` is not set")]
+    MissingEnvVar(&'static str),
+}

components/spider-proto-rust/src/generated/storage.rs

@@ -134,10 +134,8 @@ pub struct ReportTaskFailureRequest {
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct AddResourceGroupRequest {
-    #[prost(string, tag = "1")]
-    pub external_resource_group_id: ::prost::alloc::string::String,
-    #[prost(bytes = "vec", tag = "2")]
-    pub password: ::prost::alloc::vec::Vec<u8>,
+    #[prost(message, optional, tag = "1")]
+    pub credentials: ::core::option::Option<ExternalResourceGroupCredentials>,
 }
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ResourceGroupIdResponse {

components/spider-proto-rust/src/lib.rs

@@ -5,6 +5,7 @@ pub mod error;
 pub mod id;
 pub mod io;
 pub mod job;
+pub mod resource_group;
 pub mod scheduler_registration;
 pub mod unpack;
 

components/spider-proto-rust/src/resource_group.rs

@@ -0,0 +1,48 @@
+//! Conversions between protobuf and Spider core resource group types.
+
+use spider_core::types::resource_group::ExternalResourceGroupCredentials;
+
+use crate::storage;
+
+impl From<&ExternalResourceGroupCredentials> for storage::ExternalResourceGroupCredentials {
+    fn from(credentials: &ExternalResourceGroupCredentials) -> Self {
+        Self {
+            external_resource_group_id: credentials.get_external_resource_group_id().to_owned(),
+            password: credentials.get_password().to_vec(),
+        }
+    }
+}
+
+impl From<storage::ExternalResourceGroupCredentials> for ExternalResourceGroupCredentials {
+    fn from(credentials: storage::ExternalResourceGroupCredentials) -> Self {
+        Self::new(credentials.external_resource_group_id, credentials.password)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use prost::Message;
+    use spider_core::types::resource_group::ExternalResourceGroupCredentials;
+
+    use crate::storage;
+
+    #[test]
+    fn test_external_resource_group_credentials_protocol_round_trip() {
+        let credentials = ExternalResourceGroupCredentials::new(
+            "external-resource-group".to_owned(),
+            vec![0, 1, 2, 255],
+        );
+
+        let encoded = storage::ExternalResourceGroupCredentials::from(&credentials).encode_to_vec();
+        let decoded = ExternalResourceGroupCredentials::from(
+            storage::ExternalResourceGroupCredentials::decode(encoded.as_slice())
+                .expect("external resource group credentials should decode"),
+        );
+
+        assert_eq!(
+            decoded.get_external_resource_group_id(),
+            credentials.get_external_resource_group_id()
+        );
+        assert_eq!(decoded.get_password(), credentials.get_password());
+    }
+}

components/spider-proto-rust/src/unpack/storage.rs

@@ -9,6 +9,7 @@ use spider_core::types::id::ResourceGroupId;
 use spider_core::types::id::SessionId;
 use spider_core::types::id::TaskId;
 use spider_core::types::id::TaskInstanceId;
+use spider_core::types::resource_group::ExternalResourceGroupCredentials;
 use spider_utils::config::Host;
 use tonic::Code;
 
@@ -142,15 +143,14 @@ impl RequestUnpack for ReportTaskFailureRequest {
     }
 }
 
-/// Unpacks [`AddResourceGroupRequest`] into a tuple containing:
-///
-/// * The external resource group ID.
-/// * The password.
+/// Unpacks [`AddResourceGroupRequest`] into external resource group credentials.
 impl RequestUnpack for AddResourceGroupRequest {
-    type Unpacked = (String, Vec<u8>);
+    type Unpacked = ExternalResourceGroupCredentials;
 
     fn unpack(self) -> Result<Self::Unpacked, UnpackError> {
-        Ok((self.external_resource_group_id, self.password))
+        self.credentials
+            .map(Into::into)
+            .ok_or_else(|| invalid_argument("resource group credentials are missing".to_owned()))
     }
 }
 
@@ -171,16 +171,14 @@ impl RequestUnpack for VerifyResourceGroupRequest {
 /// * The execution manager's IP address.
 /// * The external resource group credentials, if present.
 impl RequestUnpack for RegisterExecutionManagerRequest {
-    type Unpacked = (IpAddr, Option<(String, Vec<u8>)>);
+    type Unpacked = (IpAddr, Option<ExternalResourceGroupCredentials>);
 
     fn unpack(self) -> Result<Self::Unpacked, UnpackError> {
         let ip_address = self
             .ip_address
             .parse::<IpAddr>()
             .map_err(|error| invalid_argument(format!("invalid IP address: {error}")))?;
-        let resource_group_credentials = self
-            .resource_group_credentials
-            .map(|credentials| (credentials.external_resource_group_id, credentials.password));
+        let resource_group_credentials = self.resource_group_credentials.map(Into::into);
         Ok((ip_address, resource_group_credentials))
     }
 }

components/spider-proto/storage/storage.proto

@@ -139,8 +139,7 @@ message ReportTaskFailureRequest {
 }
 
 message AddResourceGroupRequest {
-  string external_resource_group_id = 1;
-  bytes password = 2;
+  ExternalResourceGroupCredentials credentials = 1;
 }
 
 message ResourceGroupIdResponse {

components/spider-storage/src/db/mariadb.rs

@@ -11,6 +11,7 @@ use spider_core::types::id::SchedulerId;
 use spider_core::types::id::SessionId;
 use spider_core::types::io::SerializedTaskOutputs;
 use spider_core::types::io::TaskOutput;
+use spider_core::types::resource_group::ExternalResourceGroupCredentials;
 use spider_core::types::scheduler::RegisteredScheduler;
 use spider_derive::MySqlEnum;
 use spider_utils::config::Host;
@@ -23,7 +24,6 @@ use crate::db::DbError;
 use crate::db::DbStorage;
 use crate::db::ExecutionManagerLivenessManagement;
 use crate::db::ExternalJobOrchestration;
-use crate::db::ExternalResourceGroupCredentials;
 use crate::db::InternalJobOrchestration;
 use crate::db::RecoverableJobContext;
 use crate::db::ResourceGroupManagement;
@@ -439,21 +439,19 @@ impl ResourceGroupManagement for MariaDbStorageConnector {
             table = RESOURCE_GROUPS_TABLE_NAME,
         );
 
-        let ExternalResourceGroupCredentials {
-            external_resource_group_id,
-            password,
-        } = credentials;
         let resource_group_id = sqlx::query(QUERY)
-            .bind(&external_resource_group_id)
-            .bind(password)
+            .bind(credentials.get_external_resource_group_id())
+            .bind(credentials.get_password())
             .execute(&self.pool)
             .await
             .map_err(|e| match e {
                 sqlx::Error::Database(e)
                     if e.try_downcast_ref::<MySqlDatabaseError>()
                         .is_some_and(|mysql_err| mysql_err.number() == MYSQL_ER_DUP_ENTRY) =>
                 {
-                    DbError::ResourceGroupAlreadyExists(external_resource_group_id)
+                    DbError::ResourceGroupAlreadyExists(
+                        credentials.get_external_resource_group_id().to_owned(),
+                    )
                 }
                 e => e.into(),
             })?
@@ -516,20 +514,19 @@ impl ExecutionManagerLivenessManagement for MariaDbStorageConnector {
             table = RESOURCE_GROUPS_TABLE_NAME,
         );
 
-        let resource_group_id = if let Some(ExternalResourceGroupCredentials {
-            external_resource_group_id,
-            password,
-        }) = resource_group_credentials
-        {
+        let resource_group_id = if let Some(credentials) = resource_group_credentials {
             let resource_group_id =
                 sqlx::query_scalar::<_, ResourceGroupId>(SELECT_RESOURCE_GROUP_QUERY)
-                    .bind(&external_resource_group_id)
+                    .bind(credentials.get_external_resource_group_id())
                     .fetch_optional(&self.pool)
                     .await?
                     .ok_or_else(|| {
-                        DbError::ExternalResourceGroupNotFound(external_resource_group_id)
+                        DbError::ExternalResourceGroupNotFound(
+                            credentials.get_external_resource_group_id().to_owned(),
+                        )
                     })?;
-            ResourceGroupManagement::verify(self, resource_group_id, &password).await?;
+            ResourceGroupManagement::verify(self, resource_group_id, credentials.get_password())
+                .await?;
             Some(resource_group_id)
         } else {
             None

components/spider-storage/src/db/mod.rs

@@ -7,9 +7,9 @@ pub use mariadb::MariaDbStorageConnector;
 pub use protocol::DbStorage;
 pub use protocol::ExecutionManagerLivenessManagement;
 pub use protocol::ExternalJobOrchestration;
-pub use protocol::ExternalResourceGroupCredentials;
 pub use protocol::InternalJobOrchestration;
 pub use protocol::RecoverableJobContext;
 pub use protocol::ResourceGroupManagement;
 pub use protocol::SchedulerRegistrationManagement;
 pub use protocol::SessionManagement;
+pub use spider_core::types::resource_group::ExternalResourceGroupCredentials;