feat: Add CSV source

Cargo.toml

@@ -26,6 +26,8 @@ url = "2.5.2"
 uuid = { version = "1.10.0", features = ["v5"] }
 zitadel-rust-client = { git = "https://github.com/famedly/zitadel-rust-client", version = "0.1.0" }
 wiremock = "0.6.2"
+csv = "1.3.0"
+tempfile = "3.12.0"
 
 [dependencies.tonic]
 version = "*"
@@ -134,4 +136,3 @@ unwrap_used = "warn"
 used_underscore_binding = "warn"
 useless_let_if_seq = "warn"
 verbose_file_reads = "warn"
-

README.md

@@ -4,6 +4,7 @@ This tool synchronizes users from different sources to Famedly's Zitadel instanc
 
 Currently supported sources:
 - LDAP
+- CSV
 - Custom endpoint provided by UKT
 
 ## Configuration

config.sample.yaml

@@ -107,3 +107,11 @@ sources:
     scope: "openid read-maillist"
     # Grant type
     grant_type: client_credentials
+
+  # Configuration for the CSV source - reads a CSV file
+  # and creates **new** users in Famedly's Zitadel.
+  # Expected structure of the CSV file is as follows:
+  # email,first_name,last_name,phone
+  csv:
+    # Path to the CSV file to read from.
+    file_path:  ./tests/environment/files/test-users.csv

src/config.rs

@@ -11,6 +11,7 @@ use url::Url;
 
 use crate::{
 	sources::{
+		csv::{CsvSource, CsvSourceConfig},
 		ldap::{LdapSource, LdapSourceConfig},
 		ukt::{UktSource, UktSourceConfig},
 		Source,
@@ -44,6 +45,8 @@ pub struct SourcesConfig {
 	pub ldap: Option<LdapSourceConfig>,
 	/// Optional UKT configuration
 	pub ukt: Option<UktSourceConfig>,
+	/// Optional CSV configuration
+	pub csv: Option<CsvSourceConfig>,
 }
 
 impl Config {
@@ -95,6 +98,11 @@ impl Config {
 			sources.push(Box::new(ukt));
 		}
 
+		if let Some(csv_config) = &self.sources.csv {
+			let csv = CsvSource::new(csv_config.clone());
+			sources.push(Box::new(csv));
+		}
+
 		// Setup Zitadel client
 		let zitadel = Zitadel::new(self).await?;
 
@@ -196,7 +204,7 @@ mod tests {
           organization_id: 1
           project_id: 1
           idp_id: 1
-        
+
         sources:
           test: 1
 

src/lib.rs

@@ -6,4 +6,7 @@ mod user;
 mod zitadel;
 
 pub use config::{Config, FeatureFlag};
-pub use sources::{ldap::AttributeMapping, ukt::test_helpers};
+pub use sources::{
+	csv::test_helpers as csv_test_helpers, ldap::AttributeMapping,
+	ukt::test_helpers as ukt_test_helpers,
+};

src/sources.rs

@@ -5,6 +5,7 @@ use async_trait::async_trait;
 
 use crate::zitadel::SourceDiff;
 
+pub mod csv;
 pub mod ldap;
 pub mod ukt;
 

src/sources/csv.rs

@@ -0,0 +1,266 @@
+//! CSV source for syncing with Famedly's Zitadel.
+
+use std::{fs, path::PathBuf};
+
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use csv::Reader;
+use serde::Deserialize;
+
+use super::Source;
+use crate::{user::User, zitadel::SourceDiff};
+
+/// CSV Source
+pub struct CsvSource {
+	/// CSV Source configuration
+	csv_config: CsvSourceConfig,
+}
+
+#[async_trait]
+impl Source for CsvSource {
+	fn get_name(&self) -> &'static str {
+		"CSV"
+	}
+
+	async fn get_diff(&self) -> Result<SourceDiff> {
+		let new_users = self.read_csv()?;
+		// TODO: Implement changed and deleted users
+		// Holding off on this until we get rid of the cache concept
+		// https://github.com/famedly/ldap-sync/issues/53
+		return Ok(SourceDiff { new_users, changed_users: vec![], deleted_user_ids: vec![] });
+	}
+}
+
+impl CsvSource {
+	/// Create a new CSV source
+	pub fn new(csv_config: CsvSourceConfig) -> Self {
+		Self { csv_config }
+	}
+
+	/// Get list of users from CSV file
+	fn read_csv(&self) -> Result<Vec<User>> {
+		let file_path = &self.csv_config.file_path;
+		let file = fs::File::open(&self.csv_config.file_path)
+			.context(format!("Failed to open CSV file {}", file_path.to_string_lossy()))?;
+		let mut reader = Reader::from_reader(file);
+		Ok(reader
+			.deserialize()
+			.map(|r| r.inspect_err(|x| tracing::error!("Failed to deserialize: {x}")))
+			.filter_map(Result::ok)
+			.map(CsvData::to_user)
+			.collect())
+	}
+}
+
+/// Configuration to get a list of users from a CSV file
+#[derive(Debug, Clone, Deserialize, PartialEq)]
+pub struct CsvSourceConfig {
+	/// The path to the CSV file
+	pub file_path: PathBuf,
+}
+
+/// CSV data structure
+#[derive(Debug, Deserialize)]
+struct CsvData {
+	/// The user's email address
+	email: String,
+	/// The user's first name
+	first_name: String,
+	/// The user's last name
+	last_name: String,
+	/// The user's phone number
+	phone: String,
+}
+
+impl CsvData {
+	/// Convert CsvData to User data
+	fn to_user(csv_data: CsvData) -> User {
+		User {
+			email: csv_data.email.clone().into(),
+			first_name: csv_data.first_name.into(),
+			last_name: csv_data.last_name.into(),
+			phone: if csv_data.phone.is_empty() { None } else { Some(csv_data.phone.into()) },
+			preferred_username: csv_data.email.clone().into(),
+			external_user_id: csv_data.email.into(),
+			enabled: true,
+		}
+	}
+}
+
+/// Helper module for unit and e2e tests
+pub mod test_helpers {
+	use std::fs::write;
+
+	use anyhow::Result;
+	use tempfile::NamedTempFile;
+
+	use crate::Config;
+
+	/// Prepare a temporary CSV file with the given content and update the
+	/// config to use it as the CSV source file.
+	pub fn temp_csv_file(config: &mut Config, csv_content: &str) -> Result<NamedTempFile> {
+		let temp_file = NamedTempFile::new()?;
+		write(temp_file.path(), csv_content)?;
+
+		if let Some(csv) = config.sources.csv.as_mut() {
+			csv.file_path = temp_file.path().to_path_buf();
+		}
+
+		Ok(temp_file)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+
+	use indoc::indoc;
+
+	use super::*;
+	use crate::{user::StringOrBytes, Config};
+
+	const EXAMPLE_CONFIG: &str = indoc! {r#"
+        zitadel:
+          url: http://localhost:8080
+          key_file: tests/environment/zitadel/service-user.json
+          organization_id: 1
+          project_id: 1
+          idp_id: 1
+
+        sources:
+          csv:
+            file_path: ./test_users.csv
+
+        feature_flags: [verify_phone]
+    "#};
+
+	fn load_config() -> Config {
+		serde_yaml::from_str(EXAMPLE_CONFIG).expect("invalid config")
+	}
+
+	#[test]
+	fn test_get_users() {
+		let mut config = load_config();
+		let csv_content = indoc! {r#"
+          email,first_name,last_name,phone
+          [email protected],John,Doe,+1111111111
+          [email protected],Jane,Smith,+2222222222
+          [email protected],Alice,Johnson,
+          [email protected],Bob,Williams,+4444444444
+        "#};
+		let _file = test_helpers::temp_csv_file(&mut config, csv_content);
+
+		let csv_config = config.sources.csv.expect("CsvSource configuration is missing");
+		let csv = CsvSource::new(csv_config);
+
+		let result = csv.read_csv();
+		assert!(result.is_ok(), "Failed to get users: {:?}", result);
+
+		let users = result.expect("Failed to get users");
+		assert_eq!(users.len(), 4, "Unexpected number of users");
+		assert_eq!(
+			users[0].first_name,
+			StringOrBytes::String("John".to_owned()),
+			"Unexpected first name at index 0"
+		);
+		assert_eq!(
+			users[0].email,
+			StringOrBytes::String("[email protected]".to_owned()),
+			"Unexpected email at index 0"
+		);
+		assert_eq!(
+			users[3].last_name,
+			StringOrBytes::String("Williams".to_owned()),
+			"Unexpected last name at index 3"
+		);
+		assert_eq!(users[2].phone, None, "Unexpected phone at index 2");
+		assert_eq!(
+			users[3].phone,
+			Some(StringOrBytes::String("+4444444444".to_owned())),
+			"Unexpected phone at index 3"
+		);
+	}
+
+	#[test]
+	fn test_get_users_empty_file() {
+		let mut config = load_config();
+		let csv_content = indoc! {r#"
+          email,first_name,last_name,phone
+        "#};
+		let _file = test_helpers::temp_csv_file(&mut config, csv_content);
+
+		let csv_config = config.sources.csv.expect("CsvSource configuration is missing");
+		let csv = CsvSource::new(csv_config);
+
+		let result = csv.read_csv();
+		assert!(result.is_ok(), "Failed to get users: {:?}", result);
+
+		let users = result.expect("Failed to get users");
+		assert_eq!(users.len(), 0, "Expected empty user list");
+	}
+
+	#[test]
+	fn test_get_users_invalid_file() {
+		let mut config = load_config();
+		if let Some(csv) = config.sources.csv.as_mut() {
+			csv.file_path = PathBuf::from("invalid_path.csv");
+		}
+
+		let csv_config = config.sources.csv.expect("CsvSource configuration is missing");
+		let csv = CsvSource::new(csv_config);
+
+		let result = csv.read_csv();
+		let error = result.expect_err("Expected error for invalid CSV data");
+		assert!(
+			error.to_string().contains("Failed to open CSV file"),
+			"Unexpected error message: {:?}",
+			error
+		);
+	}
+
+	#[test]
+	fn test_get_users_invalid_headers() {
+		let mut config = load_config();
+		let csv_content = indoc! {r#"
+          first_name
+          [email protected],John,Doe,+1111111111
+        "#};
+		let _file = test_helpers::temp_csv_file(&mut config, csv_content);
+
+		let csv_config = config.sources.csv.expect("CsvSource configuration is missing");
+		let csv = CsvSource::new(csv_config);
+
+		let result = csv.read_csv();
+		let users = result.expect("Failed to get users");
+		assert_eq!(users.len(), 0, "Unexpected number of users");
+	}
+
+	#[test]
+	fn test_get_users_invalid_content() {
+		let mut config = load_config();
+		let csv_content = indoc! {r#"
+          email,first_name,last_name,phone
+          [email protected]
+          [email protected],Jane,Smith,+2222222222
+        "#};
+		let _file = test_helpers::temp_csv_file(&mut config, csv_content);
+
+		let csv_config = config.sources.csv.expect("CsvSource configuration is missing");
+		let csv = CsvSource::new(csv_config);
+
+		let result = csv.read_csv();
+		assert!(result.is_ok(), "Failed to get users: {:?}", result);
+
+		let users = result.expect("Failed to get users");
+		assert_eq!(users.len(), 1, "Unexpected number of users");
+		assert_eq!(
+			users[0].email,
+			StringOrBytes::String("[email protected]".to_owned()),
+			"Unexpected email at index 0"
+		);
+		assert_eq!(
+			users[0].last_name,
+			StringOrBytes::String("Smith".to_owned()),
+			"Unexpected last name at index 0"
+		);
+	}
+}