Refactor: Genericize default organization to 'system' and update tenant paths
This commit is contained in:
parent
7e69ab26bb
commit
b113267aef
7 changed files with 111 additions and 145 deletions
|
|
@ -231,7 +231,7 @@ When configuring CI/CD pipelines (e.g., Forgejo Actions):
|
||||||
- name: Setup Workspace
|
- name: Setup Workspace
|
||||||
run: |
|
run: |
|
||||||
# 1. Clone only the root workspace configuration
|
# 1. Clone only the root workspace configuration
|
||||||
git clone --depth 1 https://alm.pragmatismo.com.br/GeneralBots/gb.git workspace
|
git clone --depth 1 <your-git-repo-url> workspace
|
||||||
|
|
||||||
# 2. Setup only the necessary dependencies (botlib)
|
# 2. Setup only the necessary dependencies (botlib)
|
||||||
cd workspace
|
cd workspace
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ done
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# Directory to analyze
|
# Directory to analyze
|
||||||
TARGET_DIR="/opt/gbo/tenants/pragmatismo"
|
TARGET_DIR="/opt/gbo/tenants/system"
|
||||||
|
|
||||||
echo "Calculating sizes for directories in $TARGET_DIR..."
|
echo "Calculating sizes for directories in $TARGET_DIR..."
|
||||||
echo ""
|
echo ""
|
||||||
|
|
|
||||||
|
|
@ -436,7 +436,8 @@ impl PackageManager {
|
||||||
"VAULT_ADDR=http://127.0.0.1:8200 /opt/gbo/bin/vault operator unseal {}",
|
"VAULT_ADDR=http://127.0.0.1:8200 /opt/gbo/bin/vault operator unseal {}",
|
||||||
key_str
|
key_str
|
||||||
);
|
);
|
||||||
let unseal_output = safe_lxc(&["exec", container_name, "--", "bash", "-c", &unseal_cmd]);
|
let unseal_output =
|
||||||
|
safe_lxc(&["exec", container_name, "--", "bash", "-c", &unseal_cmd]);
|
||||||
|
|
||||||
if let Some(output) = unseal_output {
|
if let Some(output) = unseal_output {
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
|
|
@ -594,7 +595,7 @@ Store credentials in Vault:
|
||||||
API: http://{}:8086
|
API: http://{}:8086
|
||||||
|
|
||||||
Store credentials in Vault:
|
Store credentials in Vault:
|
||||||
botserver vault put gbo/observability url=http://{}:8086 token=<influx-token> org=pragmatismo bucket=metrics",
|
botserver vault put gbo/observability url=http://{}:8086 token=<influx-token> org=system bucket=metrics",
|
||||||
ip, ip
|
ip, ip
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
@ -926,7 +927,11 @@ Store credentials in Vault:
|
||||||
let has_subdir = if list_output.status.success() {
|
let has_subdir = if list_output.status.success() {
|
||||||
let contents = String::from_utf8_lossy(&list_output.stdout);
|
let contents = String::from_utf8_lossy(&list_output.stdout);
|
||||||
// If first entry contains '/', there's a subdirectory structure
|
// If first entry contains '/', there's a subdirectory structure
|
||||||
contents.lines().next().map(|l| l.contains('/')).unwrap_or(false)
|
contents
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.map(|l| l.contains('/'))
|
||||||
|
.unwrap_or(false)
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
};
|
};
|
||||||
|
|
@ -1081,11 +1086,9 @@ Store credentials in Vault:
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to set env: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("Failed to set env: {}", e))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = cmd
|
let output = cmd.execute().with_context(|| {
|
||||||
.execute()
|
format!("Failed to execute command for component '{}'", component)
|
||||||
.with_context(|| {
|
})?;
|
||||||
format!("Failed to execute command for component '{}'", component)
|
|
||||||
})?;
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
error!(
|
error!(
|
||||||
"Command had non-zero exit: {}",
|
"Command had non-zero exit: {}",
|
||||||
|
|
@ -1269,7 +1272,8 @@ Store credentials in Vault:
|
||||||
"proxy",
|
"proxy",
|
||||||
&listen_arg,
|
&listen_arg,
|
||||||
&connect_arg,
|
&connect_arg,
|
||||||
]).ok_or_else(|| anyhow::anyhow!("Failed to execute lxc port forward command"))?;
|
])
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("Failed to execute lxc port forward command"))?;
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
warn!("Failed to setup port forwarding for port {}", port);
|
warn!("Failed to setup port forwarding for port {}", port);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -860,7 +860,7 @@ impl PackageManager {
|
||||||
"mkdir -p {{CONF_PATH}}/influxdb".to_string(),
|
"mkdir -p {{CONF_PATH}}/influxdb".to_string(),
|
||||||
],
|
],
|
||||||
post_install_cmds_linux: vec![
|
post_install_cmds_linux: vec![
|
||||||
"{{BIN_PATH}}/influx setup --org pragmatismo --bucket metrics --username admin --password {{GENERATED_PASSWORD}} --force".to_string(),
|
"{{BIN_PATH}}/influx setup --org system --bucket metrics --username admin --password {{GENERATED_PASSWORD}} --force".to_string(),
|
||||||
],
|
],
|
||||||
pre_install_cmds_macos: vec![
|
pre_install_cmds_macos: vec![
|
||||||
"mkdir -p {{DATA_PATH}}/influxdb".to_string(),
|
"mkdir -p {{DATA_PATH}}/influxdb".to_string(),
|
||||||
|
|
@ -1082,7 +1082,8 @@ EOF"#.to_string(),
|
||||||
|
|
||||||
trace!(
|
trace!(
|
||||||
"Starting component {} with command: {}",
|
"Starting component {} with command: {}",
|
||||||
component.name, rendered_cmd
|
component.name,
|
||||||
|
rendered_cmd
|
||||||
);
|
);
|
||||||
trace!(
|
trace!(
|
||||||
"Working directory: {}, logs_path: {}",
|
"Working directory: {}, logs_path: {}",
|
||||||
|
|
@ -1108,7 +1109,8 @@ EOF"#.to_string(),
|
||||||
|
|
||||||
trace!(
|
trace!(
|
||||||
"About to spawn shell command for {}: {}",
|
"About to spawn shell command for {}: {}",
|
||||||
component.name, rendered_cmd
|
component.name,
|
||||||
|
rendered_cmd
|
||||||
);
|
);
|
||||||
trace!("[START] Working dir: {}", bin_path.display());
|
trace!("[START] Working dir: {}", bin_path.display());
|
||||||
let child = SafeCommand::new("sh")
|
let child = SafeCommand::new("sh")
|
||||||
|
|
@ -1118,11 +1120,7 @@ EOF"#.to_string(),
|
||||||
.and_then(|cmd| cmd.spawn_with_envs(&evaluated_envs))
|
.and_then(|cmd| cmd.spawn_with_envs(&evaluated_envs))
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to spawn process: {}", e));
|
.map_err(|e| anyhow::anyhow!("Failed to spawn process: {}", e));
|
||||||
|
|
||||||
trace!(
|
trace!("Spawn result for {}: {:?}", component.name, child.is_ok());
|
||||||
"Spawn result for {}: {:?}",
|
|
||||||
component.name,
|
|
||||||
child.is_ok()
|
|
||||||
);
|
|
||||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||||
|
|
||||||
trace!(
|
trace!(
|
||||||
|
|
@ -1132,11 +1130,7 @@ EOF"#.to_string(),
|
||||||
let check_proc = safe_pgrep(&["-f", &component.name]);
|
let check_proc = safe_pgrep(&["-f", &component.name]);
|
||||||
if let Some(output) = check_proc {
|
if let Some(output) = check_proc {
|
||||||
let pids = String::from_utf8_lossy(&output.stdout);
|
let pids = String::from_utf8_lossy(&output.stdout);
|
||||||
trace!(
|
trace!("pgrep '{}' result: '{}'", component.name, pids.trim());
|
||||||
"pgrep '{}' result: '{}'",
|
|
||||||
component.name,
|
|
||||||
pids.trim()
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
match child {
|
match child {
|
||||||
|
|
@ -1199,11 +1193,14 @@ EOF"#.to_string(),
|
||||||
client_key.display(),
|
client_key.display(),
|
||||||
vault_addr
|
vault_addr
|
||||||
))
|
))
|
||||||
.map(|o| o.status.success())
|
.map(|o| o.status.success())
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
|
||||||
if !vault_check {
|
if !vault_check {
|
||||||
trace!("Vault not reachable at {}, skipping credential fetch", vault_addr);
|
trace!(
|
||||||
|
"Vault not reachable at {}, skipping credential fetch",
|
||||||
|
vault_addr
|
||||||
|
);
|
||||||
return credentials;
|
return credentials;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1211,10 +1208,18 @@ EOF"#.to_string(),
|
||||||
let vault_bin_str = vault_bin.to_string_lossy();
|
let vault_bin_str = vault_bin.to_string_lossy();
|
||||||
|
|
||||||
// Get CA cert path for Vault TLS
|
// Get CA cert path for Vault TLS
|
||||||
let ca_cert_path = std::env::var("VAULT_CACERT")
|
let ca_cert_path = std::env::var("VAULT_CACERT").unwrap_or_else(|_| {
|
||||||
.unwrap_or_else(|_| base_path.join("conf/system/certificates/ca/ca.crt").to_string_lossy().to_string());
|
base_path
|
||||||
|
.join("conf/system/certificates/ca/ca.crt")
|
||||||
|
.to_string_lossy()
|
||||||
|
.to_string()
|
||||||
|
});
|
||||||
|
|
||||||
trace!("Fetching drive credentials from Vault at {} using {}", vault_addr, vault_bin_str);
|
trace!(
|
||||||
|
"Fetching drive credentials from Vault at {} using {}",
|
||||||
|
vault_addr,
|
||||||
|
vault_bin_str
|
||||||
|
);
|
||||||
let drive_cmd = format!(
|
let drive_cmd = format!(
|
||||||
"VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} {} kv get -format=json secret/gbo/drive",
|
"VAULT_ADDR={} VAULT_TOKEN={} VAULT_CACERT={} {} kv get -format=json secret/gbo/drive",
|
||||||
vault_addr, vault_token, ca_cert_path, vault_bin_str
|
vault_addr, vault_token, ca_cert_path, vault_bin_str
|
||||||
|
|
@ -1227,13 +1232,19 @@ EOF"#.to_string(),
|
||||||
match serde_json::from_str::<serde_json::Value>(&json_str) {
|
match serde_json::from_str::<serde_json::Value>(&json_str) {
|
||||||
Ok(json) => {
|
Ok(json) => {
|
||||||
if let Some(data) = json.get("data").and_then(|d| d.get("data")) {
|
if let Some(data) = json.get("data").and_then(|d| d.get("data")) {
|
||||||
if let Some(accesskey) = data.get("accesskey").and_then(|v| v.as_str()) {
|
if let Some(accesskey) =
|
||||||
|
data.get("accesskey").and_then(|v| v.as_str())
|
||||||
|
{
|
||||||
trace!("Found DRIVE_ACCESSKEY from Vault");
|
trace!("Found DRIVE_ACCESSKEY from Vault");
|
||||||
credentials.insert("DRIVE_ACCESSKEY".to_string(), accesskey.to_string());
|
credentials.insert(
|
||||||
|
"DRIVE_ACCESSKEY".to_string(),
|
||||||
|
accesskey.to_string(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if let Some(secret) = data.get("secret").and_then(|v| v.as_str()) {
|
if let Some(secret) = data.get("secret").and_then(|v| v.as_str()) {
|
||||||
trace!("Found DRIVE_SECRET from Vault");
|
trace!("Found DRIVE_SECRET from Vault");
|
||||||
credentials.insert("DRIVE_SECRET".to_string(), secret.to_string());
|
credentials
|
||||||
|
.insert("DRIVE_SECRET".to_string(), secret.to_string());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
warn!("Vault response missing data.data field");
|
warn!("Vault response missing data.data field");
|
||||||
|
|
@ -1259,7 +1270,8 @@ EOF"#.to_string(),
|
||||||
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&json_str) {
|
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&json_str) {
|
||||||
if let Some(data) = json.get("data").and_then(|d| d.get("data")) {
|
if let Some(data) = json.get("data").and_then(|d| d.get("data")) {
|
||||||
if let Some(password) = data.get("password").and_then(|v| v.as_str()) {
|
if let Some(password) = data.get("password").and_then(|v| v.as_str()) {
|
||||||
credentials.insert("CACHE_PASSWORD".to_string(), password.to_string());
|
credentials
|
||||||
|
.insert("CACHE_PASSWORD".to_string(), password.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,9 +12,8 @@ use std::sync::RwLock;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
/// Global product configuration instance
|
/// Global product configuration instance
|
||||||
pub static PRODUCT_CONFIG: Lazy<RwLock<ProductConfig>> = Lazy::new(|| {
|
pub static PRODUCT_CONFIG: Lazy<RwLock<ProductConfig>> =
|
||||||
RwLock::new(ProductConfig::load().unwrap_or_default())
|
Lazy::new(|| RwLock::new(ProductConfig::load().unwrap_or_default()));
|
||||||
});
|
|
||||||
|
|
||||||
/// Product configuration structure
|
/// Product configuration structure
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|
@ -52,9 +51,22 @@ impl Default for ProductConfig {
|
||||||
let mut apps = HashSet::new();
|
let mut apps = HashSet::new();
|
||||||
// All apps enabled by default
|
// All apps enabled by default
|
||||||
for app in &[
|
for app in &[
|
||||||
"chat", "mail", "calendar", "drive", "tasks", "docs", "paper",
|
"chat",
|
||||||
"sheet", "slides", "meet", "research", "sources", "analytics",
|
"mail",
|
||||||
"admin", "monitoring", "settings",
|
"calendar",
|
||||||
|
"drive",
|
||||||
|
"tasks",
|
||||||
|
"docs",
|
||||||
|
"paper",
|
||||||
|
"sheet",
|
||||||
|
"slides",
|
||||||
|
"meet",
|
||||||
|
"research",
|
||||||
|
"sources",
|
||||||
|
"analytics",
|
||||||
|
"admin",
|
||||||
|
"monitoring",
|
||||||
|
"settings",
|
||||||
] {
|
] {
|
||||||
apps.insert(app.to_string());
|
apps.insert(app.to_string());
|
||||||
}
|
}
|
||||||
|
|
@ -67,7 +79,7 @@ impl Default for ProductConfig {
|
||||||
favicon: None,
|
favicon: None,
|
||||||
primary_color: None,
|
primary_color: None,
|
||||||
support_email: None,
|
support_email: None,
|
||||||
docs_url: Some("https://docs.pragmatismo.com.br".to_string()),
|
docs_url: None,
|
||||||
copyright: None,
|
copyright: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -76,11 +88,7 @@ impl Default for ProductConfig {
|
||||||
impl ProductConfig {
|
impl ProductConfig {
|
||||||
/// Load configuration from .product file
|
/// Load configuration from .product file
|
||||||
pub fn load() -> Result<Self, ProductConfigError> {
|
pub fn load() -> Result<Self, ProductConfigError> {
|
||||||
let paths = [
|
let paths = [".product", "./botserver/.product", "../.product"];
|
||||||
".product",
|
|
||||||
"./botserver/.product",
|
|
||||||
"../.product",
|
|
||||||
];
|
|
||||||
|
|
||||||
let mut content = None;
|
let mut content = None;
|
||||||
for path in &paths {
|
for path in &paths {
|
||||||
|
|
@ -215,7 +223,9 @@ impl ProductConfig {
|
||||||
/// Get copyright text with year substitution
|
/// Get copyright text with year substitution
|
||||||
pub fn get_copyright(&self) -> String {
|
pub fn get_copyright(&self) -> String {
|
||||||
let year = chrono::Utc::now().format("%Y").to_string();
|
let year = chrono::Utc::now().format("%Y").to_string();
|
||||||
let template = self.copyright.as_deref()
|
let template = self
|
||||||
|
.copyright
|
||||||
|
.as_deref()
|
||||||
.unwrap_or("© {year} {name}. All rights reserved.");
|
.unwrap_or("© {year} {name}. All rights reserved.");
|
||||||
|
|
||||||
template
|
template
|
||||||
|
|
@ -231,7 +241,8 @@ impl ProductConfig {
|
||||||
/// Reload configuration from file
|
/// Reload configuration from file
|
||||||
pub fn reload() -> Result<(), ProductConfigError> {
|
pub fn reload() -> Result<(), ProductConfigError> {
|
||||||
let new_config = Self::load()?;
|
let new_config = Self::load()?;
|
||||||
let mut config = PRODUCT_CONFIG.write()
|
let mut config = PRODUCT_CONFIG
|
||||||
|
.write()
|
||||||
.map_err(|_| ProductConfigError::LockError)?;
|
.map_err(|_| ProductConfigError::LockError)?;
|
||||||
*config = new_config;
|
*config = new_config;
|
||||||
info!("Product configuration reloaded");
|
info!("Product configuration reloaded");
|
||||||
|
|
@ -327,7 +338,7 @@ pub fn get_product_config_json() -> serde_json::Value {
|
||||||
"compiled_features": compiled,
|
"compiled_features": compiled,
|
||||||
"version": env!("CARGO_PKG_VERSION"),
|
"version": env!("CARGO_PKG_VERSION"),
|
||||||
"theme": "sentient",
|
"theme": "sentient",
|
||||||
})
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -337,7 +348,6 @@ pub fn get_workspace_manifest() -> serde_json::Value {
|
||||||
serde_json::to_value(manifest).unwrap_or_else(|_| serde_json::json!({}))
|
serde_json::to_value(manifest).unwrap_or_else(|_| serde_json::json!({}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Middleware to check if an app is enabled before allowing API access
|
/// Middleware to check if an app is enabled before allowing API access
|
||||||
pub async fn app_gate_middleware(
|
pub async fn app_gate_middleware(
|
||||||
req: axum::http::Request<axum::body::Body>,
|
req: axum::http::Request<axum::body::Body>,
|
||||||
|
|
@ -392,16 +402,13 @@ pub async fn app_gate_middleware(
|
||||||
// but here we enforce strict feature containment.
|
// but here we enforce strict feature containment.
|
||||||
// Exception: 'settings' and 'auth' are often core.
|
// Exception: 'settings' and 'auth' are often core.
|
||||||
if app != "settings" && app != "auth" && !crate::core::features::is_feature_compiled(app) {
|
if app != "settings" && app != "auth" && !crate::core::features::is_feature_compiled(app) {
|
||||||
let error_response = serde_json::json!({
|
let error_response = serde_json::json!({
|
||||||
"error": "not_implemented",
|
"error": "not_implemented",
|
||||||
"message": format!("The '{}' feature is not compiled in this build", app),
|
"message": format!("The '{}' feature is not compiled in this build", app),
|
||||||
"code": 501
|
"code": 501
|
||||||
});
|
});
|
||||||
|
|
||||||
return (
|
return (StatusCode::NOT_IMPLEMENTED, axum::Json(error_response)).into_response();
|
||||||
StatusCode::NOT_IMPLEMENTED,
|
|
||||||
axum::Json(error_response)
|
|
||||||
).into_response();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !is_app_enabled(app) {
|
if !is_app_enabled(app) {
|
||||||
|
|
@ -411,10 +418,7 @@ pub async fn app_gate_middleware(
|
||||||
"code": 403
|
"code": 403
|
||||||
});
|
});
|
||||||
|
|
||||||
return (
|
return (StatusCode::FORBIDDEN, axum::Json(error_response)).into_response();
|
||||||
StatusCode::FORBIDDEN,
|
|
||||||
axum::Json(error_response)
|
|
||||||
).into_response();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -424,9 +428,22 @@ pub async fn app_gate_middleware(
|
||||||
/// Get list of disabled apps for logging/debugging
|
/// Get list of disabled apps for logging/debugging
|
||||||
pub fn get_disabled_apps() -> Vec<String> {
|
pub fn get_disabled_apps() -> Vec<String> {
|
||||||
let all_apps = vec![
|
let all_apps = vec![
|
||||||
"chat", "mail", "calendar", "drive", "tasks", "docs", "paper",
|
"chat",
|
||||||
"sheet", "slides", "meet", "research", "sources", "analytics",
|
"mail",
|
||||||
"admin", "monitoring", "settings",
|
"calendar",
|
||||||
|
"drive",
|
||||||
|
"tasks",
|
||||||
|
"docs",
|
||||||
|
"paper",
|
||||||
|
"sheet",
|
||||||
|
"slides",
|
||||||
|
"meet",
|
||||||
|
"research",
|
||||||
|
"sources",
|
||||||
|
"analytics",
|
||||||
|
"admin",
|
||||||
|
"monitoring",
|
||||||
|
"settings",
|
||||||
];
|
];
|
||||||
|
|
||||||
all_apps
|
all_apps
|
||||||
|
|
|
||||||
|
|
@ -255,9 +255,7 @@ impl SecretsManager {
|
||||||
s.get("url")
|
s.get("url")
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_else(|| "http://localhost:8086".into()),
|
.unwrap_or_else(|| "http://localhost:8086".into()),
|
||||||
s.get("org")
|
s.get("org").cloned().unwrap_or_else(|| "system".into()),
|
||||||
.cloned()
|
|
||||||
.unwrap_or_else(|| "pragmatismo".into()),
|
|
||||||
s.get("bucket").cloned().unwrap_or_else(|| "metrics".into()),
|
s.get("bucket").cloned().unwrap_or_else(|| "metrics".into()),
|
||||||
s.get("token").cloned().unwrap_or_default(),
|
s.get("token").cloned().unwrap_or_default(),
|
||||||
))
|
))
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
use crate::shared::utils::create_tls_client;
|
use crate::shared::utils::create_tls_client;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
@ -25,10 +6,8 @@ use std::sync::Arc;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct TimeSeriesConfig {
|
pub struct TimeSeriesConfig {
|
||||||
|
|
||||||
pub url: String,
|
pub url: String,
|
||||||
|
|
||||||
pub token: String,
|
pub token: String,
|
||||||
|
|
@ -49,7 +28,7 @@ impl Default for TimeSeriesConfig {
|
||||||
Self {
|
Self {
|
||||||
url: "http://localhost:8086".to_string(),
|
url: "http://localhost:8086".to_string(),
|
||||||
token: String::new(),
|
token: String::new(),
|
||||||
org: "pragmatismo".to_string(),
|
org: "system".to_string(),
|
||||||
bucket: "metrics".to_string(),
|
bucket: "metrics".to_string(),
|
||||||
batch_size: 1000,
|
batch_size: 1000,
|
||||||
flush_interval_ms: 1000,
|
flush_interval_ms: 1000,
|
||||||
|
|
@ -58,10 +37,8 @@ impl Default for TimeSeriesConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct MetricPoint {
|
pub struct MetricPoint {
|
||||||
|
|
||||||
pub measurement: String,
|
pub measurement: String,
|
||||||
|
|
||||||
pub tags: HashMap<String, String>,
|
pub tags: HashMap<String, String>,
|
||||||
|
|
@ -71,7 +48,6 @@ pub struct MetricPoint {
|
||||||
pub timestamp: Option<DateTime<Utc>>,
|
pub timestamp: Option<DateTime<Utc>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub enum FieldValue {
|
pub enum FieldValue {
|
||||||
Float(f64),
|
Float(f64),
|
||||||
|
|
@ -82,7 +58,6 @@ pub enum FieldValue {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MetricPoint {
|
impl MetricPoint {
|
||||||
|
|
||||||
pub fn new(measurement: impl Into<String>) -> Self {
|
pub fn new(measurement: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
measurement: measurement.into(),
|
measurement: measurement.into(),
|
||||||
|
|
@ -92,55 +67,46 @@ impl MetricPoint {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn tag(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn tag(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
self.tags.insert(key.into(), value.into());
|
self.tags.insert(key.into(), value.into());
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn field_f64(mut self, key: impl Into<String>, value: f64) -> Self {
|
pub fn field_f64(mut self, key: impl Into<String>, value: f64) -> Self {
|
||||||
self.fields.insert(key.into(), FieldValue::Float(value));
|
self.fields.insert(key.into(), FieldValue::Float(value));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn field_i64(mut self, key: impl Into<String>, value: i64) -> Self {
|
pub fn field_i64(mut self, key: impl Into<String>, value: i64) -> Self {
|
||||||
self.fields.insert(key.into(), FieldValue::Integer(value));
|
self.fields.insert(key.into(), FieldValue::Integer(value));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn field_u64(mut self, key: impl Into<String>, value: u64) -> Self {
|
pub fn field_u64(mut self, key: impl Into<String>, value: u64) -> Self {
|
||||||
self.fields
|
self.fields
|
||||||
.insert(key.into(), FieldValue::UnsignedInteger(value));
|
.insert(key.into(), FieldValue::UnsignedInteger(value));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn field_str(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn field_str(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
self.fields
|
self.fields
|
||||||
.insert(key.into(), FieldValue::String(value.into()));
|
.insert(key.into(), FieldValue::String(value.into()));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn field_bool(mut self, key: impl Into<String>, value: bool) -> Self {
|
pub fn field_bool(mut self, key: impl Into<String>, value: bool) -> Self {
|
||||||
self.fields.insert(key.into(), FieldValue::Boolean(value));
|
self.fields.insert(key.into(), FieldValue::Boolean(value));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn at(mut self, timestamp: DateTime<Utc>) -> Self {
|
pub fn at(mut self, timestamp: DateTime<Utc>) -> Self {
|
||||||
self.timestamp = Some(timestamp);
|
self.timestamp = Some(timestamp);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn to_line_protocol(&self) -> String {
|
pub fn to_line_protocol(&self) -> String {
|
||||||
let mut line = self.measurement.clone();
|
let mut line = self.measurement.clone();
|
||||||
|
|
||||||
|
|
||||||
let mut sorted_tags: Vec<_> = self.tags.iter().collect();
|
let mut sorted_tags: Vec<_> = self.tags.iter().collect();
|
||||||
sorted_tags.sort_by_key(|(k, _)| *k);
|
sorted_tags.sort_by_key(|(k, _)| *k);
|
||||||
for (key, value) in sorted_tags {
|
for (key, value) in sorted_tags {
|
||||||
|
|
@ -150,7 +116,6 @@ impl MetricPoint {
|
||||||
line.push_str(&escape_tag_value(value));
|
line.push_str(&escape_tag_value(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
line.push(' ');
|
line.push(' ');
|
||||||
let mut sorted_fields: Vec<_> = self.fields.iter().collect();
|
let mut sorted_fields: Vec<_> = self.fields.iter().collect();
|
||||||
sorted_fields.sort_by_key(|(k, _)| *k);
|
sorted_fields.sort_by_key(|(k, _)| *k);
|
||||||
|
|
@ -171,7 +136,6 @@ impl MetricPoint {
|
||||||
.collect();
|
.collect();
|
||||||
line.push_str(&fields_str.join(","));
|
line.push_str(&fields_str.join(","));
|
||||||
|
|
||||||
|
|
||||||
if let Some(ts) = self.timestamp {
|
if let Some(ts) = self.timestamp {
|
||||||
line.push(' ');
|
line.push(' ');
|
||||||
line.push_str(&ts.timestamp_nanos_opt().unwrap_or(0).to_string());
|
line.push_str(&ts.timestamp_nanos_opt().unwrap_or(0).to_string());
|
||||||
|
|
@ -181,40 +145,34 @@ impl MetricPoint {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn escape_tag_key(s: &str) -> String {
|
fn escape_tag_key(s: &str) -> String {
|
||||||
s.replace(',', "\\,")
|
s.replace(',', "\\,")
|
||||||
.replace('=', "\\=")
|
.replace('=', "\\=")
|
||||||
.replace(' ', "\\ ")
|
.replace(' ', "\\ ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn escape_tag_value(s: &str) -> String {
|
fn escape_tag_value(s: &str) -> String {
|
||||||
s.replace(',', "\\,")
|
s.replace(',', "\\,")
|
||||||
.replace('=', "\\=")
|
.replace('=', "\\=")
|
||||||
.replace(' ', "\\ ")
|
.replace(' ', "\\ ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn escape_field_key(s: &str) -> String {
|
fn escape_field_key(s: &str) -> String {
|
||||||
s.replace(',', "\\,")
|
s.replace(',', "\\,")
|
||||||
.replace('=', "\\=")
|
.replace('=', "\\=")
|
||||||
.replace(' ', "\\ ")
|
.replace(' ', "\\ ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn escape_string_value(s: &str) -> String {
|
fn escape_string_value(s: &str) -> String {
|
||||||
s.replace('\\', "\\\\").replace('"', "\\\"")
|
s.replace('\\', "\\\\").replace('"', "\\\"")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct QueryResult {
|
pub struct QueryResult {
|
||||||
pub columns: Vec<String>,
|
pub columns: Vec<String>,
|
||||||
pub rows: Vec<Vec<serde_json::Value>>,
|
pub rows: Vec<Vec<serde_json::Value>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub struct TimeSeriesClient {
|
pub struct TimeSeriesClient {
|
||||||
config: TimeSeriesConfig,
|
config: TimeSeriesConfig,
|
||||||
http_client: reqwest::Client,
|
http_client: reqwest::Client,
|
||||||
|
|
@ -223,7 +181,6 @@ pub struct TimeSeriesClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TimeSeriesClient {
|
impl TimeSeriesClient {
|
||||||
|
|
||||||
pub async fn new(config: TimeSeriesConfig) -> Result<Self, TimeSeriesError> {
|
pub async fn new(config: TimeSeriesConfig) -> Result<Self, TimeSeriesError> {
|
||||||
let http_client = create_tls_client(Some(30));
|
let http_client = create_tls_client(Some(30));
|
||||||
|
|
||||||
|
|
@ -237,23 +194,15 @@ impl TimeSeriesClient {
|
||||||
write_sender,
|
write_sender,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
let buffer_clone = write_buffer.clone();
|
let buffer_clone = write_buffer.clone();
|
||||||
let config_clone = config.clone();
|
let config_clone = config.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
Self::background_writer(
|
Self::background_writer(write_receiver, buffer_clone, http_client, config_clone).await;
|
||||||
write_receiver,
|
|
||||||
buffer_clone,
|
|
||||||
http_client,
|
|
||||||
config_clone,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(client)
|
Ok(client)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async fn background_writer(
|
async fn background_writer(
|
||||||
mut receiver: mpsc::Receiver<MetricPoint>,
|
mut receiver: mpsc::Receiver<MetricPoint>,
|
||||||
buffer: Arc<RwLock<Vec<MetricPoint>>>,
|
buffer: Arc<RwLock<Vec<MetricPoint>>>,
|
||||||
|
|
@ -291,7 +240,6 @@ impl TimeSeriesClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async fn flush_points(
|
async fn flush_points(
|
||||||
http_client: &reqwest::Client,
|
http_client: &reqwest::Client,
|
||||||
config: &TimeSeriesConfig,
|
config: &TimeSeriesConfig,
|
||||||
|
|
@ -334,7 +282,6 @@ impl TimeSeriesClient {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn write_point(&self, point: MetricPoint) -> Result<(), TimeSeriesError> {
|
pub async fn write_point(&self, point: MetricPoint) -> Result<(), TimeSeriesError> {
|
||||||
self.write_sender
|
self.write_sender
|
||||||
.send(point)
|
.send(point)
|
||||||
|
|
@ -342,7 +289,6 @@ impl TimeSeriesClient {
|
||||||
.map_err(|e| TimeSeriesError::WriteError(e.to_string()))
|
.map_err(|e| TimeSeriesError::WriteError(e.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn write_points(&self, points: Vec<MetricPoint>) -> Result<(), TimeSeriesError> {
|
pub async fn write_points(&self, points: Vec<MetricPoint>) -> Result<(), TimeSeriesError> {
|
||||||
for point in points {
|
for point in points {
|
||||||
self.write_point(point).await?;
|
self.write_point(point).await?;
|
||||||
|
|
@ -350,7 +296,6 @@ impl TimeSeriesClient {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn query(&self, flux_query: &str) -> Result<QueryResult, TimeSeriesError> {
|
pub async fn query(&self, flux_query: &str) -> Result<QueryResult, TimeSeriesError> {
|
||||||
let url = format!("{}/api/v2/query?org={}", self.config.url, self.config.org);
|
let url = format!("{}/api/v2/query?org={}", self.config.url, self.config.org);
|
||||||
|
|
||||||
|
|
@ -382,7 +327,6 @@ impl TimeSeriesClient {
|
||||||
Self::parse_csv_result(&csv_data)
|
Self::parse_csv_result(&csv_data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fn parse_csv_result(csv_data: &str) -> Result<QueryResult, TimeSeriesError> {
|
fn parse_csv_result(csv_data: &str) -> Result<QueryResult, TimeSeriesError> {
|
||||||
let mut result = QueryResult {
|
let mut result = QueryResult {
|
||||||
columns: Vec::new(),
|
columns: Vec::new(),
|
||||||
|
|
@ -391,7 +335,6 @@ impl TimeSeriesClient {
|
||||||
|
|
||||||
let mut lines = csv_data.lines().peekable();
|
let mut lines = csv_data.lines().peekable();
|
||||||
|
|
||||||
|
|
||||||
while let Some(line) = lines.peek() {
|
while let Some(line) = lines.peek() {
|
||||||
if line.starts_with('#') || line.is_empty() {
|
if line.starts_with('#') || line.is_empty() {
|
||||||
lines.next();
|
lines.next();
|
||||||
|
|
@ -400,12 +343,13 @@ impl TimeSeriesClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if let Some(header_line) = lines.next() {
|
if let Some(header_line) = lines.next() {
|
||||||
result.columns = header_line.split(',').map(|s| s.trim().to_string()).collect();
|
result.columns = header_line
|
||||||
|
.split(',')
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for line in lines {
|
for line in lines {
|
||||||
if line.is_empty() || line.starts_with('#') {
|
if line.is_empty() || line.starts_with('#') {
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -436,7 +380,6 @@ impl TimeSeriesClient {
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn query_range(
|
pub async fn query_range(
|
||||||
&self,
|
&self,
|
||||||
measurement: &str,
|
measurement: &str,
|
||||||
|
|
@ -459,7 +402,6 @@ impl TimeSeriesClient {
|
||||||
self.query(&flux).await
|
self.query(&flux).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn query_last(&self, measurement: &str) -> Result<QueryResult, TimeSeriesError> {
|
pub async fn query_last(&self, measurement: &str) -> Result<QueryResult, TimeSeriesError> {
|
||||||
let flux = format!(
|
let flux = format!(
|
||||||
r#"from(bucket: "{}")
|
r#"from(bucket: "{}")
|
||||||
|
|
@ -472,7 +414,6 @@ impl TimeSeriesClient {
|
||||||
self.query(&flux).await
|
self.query(&flux).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn query_stats(
|
pub async fn query_stats(
|
||||||
&self,
|
&self,
|
||||||
measurement: &str,
|
measurement: &str,
|
||||||
|
|
@ -498,7 +439,6 @@ impl TimeSeriesClient {
|
||||||
self.query(&flux).await
|
self.query(&flux).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub async fn health_check(&self) -> Result<bool, TimeSeriesError> {
|
pub async fn health_check(&self) -> Result<bool, TimeSeriesError> {
|
||||||
let url = format!("{}/health", self.config.url);
|
let url = format!("{}/health", self.config.url);
|
||||||
|
|
||||||
|
|
@ -513,11 +453,9 @@ impl TimeSeriesClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub struct Metrics;
|
pub struct Metrics;
|
||||||
|
|
||||||
impl Metrics {
|
impl Metrics {
|
||||||
|
|
||||||
pub fn message(bot_id: &str, channel: &str, direction: &str) -> MetricPoint {
|
pub fn message(bot_id: &str, channel: &str, direction: &str) -> MetricPoint {
|
||||||
MetricPoint::new("messages")
|
MetricPoint::new("messages")
|
||||||
.tag("bot_id", bot_id)
|
.tag("bot_id", bot_id)
|
||||||
|
|
@ -527,7 +465,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn response_time(bot_id: &str, duration_ms: f64) -> MetricPoint {
|
pub fn response_time(bot_id: &str, duration_ms: f64) -> MetricPoint {
|
||||||
MetricPoint::new("response_time")
|
MetricPoint::new("response_time")
|
||||||
.tag("bot_id", bot_id)
|
.tag("bot_id", bot_id)
|
||||||
|
|
@ -535,7 +472,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn llm_tokens(
|
pub fn llm_tokens(
|
||||||
bot_id: &str,
|
bot_id: &str,
|
||||||
model: &str,
|
model: &str,
|
||||||
|
|
@ -551,7 +487,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn active_sessions(bot_id: &str, count: i64) -> MetricPoint {
|
pub fn active_sessions(bot_id: &str, count: i64) -> MetricPoint {
|
||||||
MetricPoint::new("active_sessions")
|
MetricPoint::new("active_sessions")
|
||||||
.tag("bot_id", bot_id)
|
.tag("bot_id", bot_id)
|
||||||
|
|
@ -559,7 +494,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn error(bot_id: &str, error_type: &str, message: &str) -> MetricPoint {
|
pub fn error(bot_id: &str, error_type: &str, message: &str) -> MetricPoint {
|
||||||
MetricPoint::new("errors")
|
MetricPoint::new("errors")
|
||||||
.tag("bot_id", bot_id)
|
.tag("bot_id", bot_id)
|
||||||
|
|
@ -569,7 +503,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn storage_usage(bot_id: &str, bytes_used: u64, file_count: u64) -> MetricPoint {
|
pub fn storage_usage(bot_id: &str, bytes_used: u64, file_count: u64) -> MetricPoint {
|
||||||
MetricPoint::new("storage_usage")
|
MetricPoint::new("storage_usage")
|
||||||
.tag("bot_id", bot_id)
|
.tag("bot_id", bot_id)
|
||||||
|
|
@ -578,8 +511,12 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn api_request(
|
||||||
pub fn api_request(endpoint: &str, method: &str, status_code: i64, duration_ms: f64) -> MetricPoint {
|
endpoint: &str,
|
||||||
|
method: &str,
|
||||||
|
status_code: i64,
|
||||||
|
duration_ms: f64,
|
||||||
|
) -> MetricPoint {
|
||||||
MetricPoint::new("api_requests")
|
MetricPoint::new("api_requests")
|
||||||
.tag("endpoint", endpoint)
|
.tag("endpoint", endpoint)
|
||||||
.tag("method", method)
|
.tag("method", method)
|
||||||
|
|
@ -589,7 +526,6 @@ impl Metrics {
|
||||||
.at(Utc::now())
|
.at(Utc::now())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn system(cpu_percent: f64, memory_percent: f64, disk_percent: f64) -> MetricPoint {
|
pub fn system(cpu_percent: f64, memory_percent: f64, disk_percent: f64) -> MetricPoint {
|
||||||
MetricPoint::new("system_metrics")
|
MetricPoint::new("system_metrics")
|
||||||
.field_f64("cpu_percent", cpu_percent)
|
.field_f64("cpu_percent", cpu_percent)
|
||||||
|
|
@ -599,7 +535,6 @@ impl Metrics {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum TimeSeriesError {
|
pub enum TimeSeriesError {
|
||||||
ConnectionError(String),
|
ConnectionError(String),
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue