GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

Add include_dir dependency and use it for embedded migrations

Browse source 
Use include_dir to embed migration scripts and load them at runtime.
This change allows for easier management and versioning of migrations.
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-10-20 09:42:07 -03:00
parent 93dab6f741
commit 30b026585d
8 changed files with 107 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
Cargo.lock generated
View file

@ -1032,6 +1032,7 @@ dependencies = [
"futures-util",
"headless_chrome",
"imap",
"include_dir",
"indicatif",
"lettre",
"livekit",
@ -2855,6 +2856,25 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
[[package]]
name = "include_dir"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
dependencies = [
"include_dir_macros",
]
[[package]]
name = "include_dir_macros"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "indexmap"
version = "1.9.3"

1
Cargo.toml
View file

@ -63,6 +63,7 @@ futures = "0.3"
futures-util = "0.3"
lettre = { version = "0.11", features = ["smtp-transport", "builder", "tokio1", "tokio1-native-tls"] }
livekit = "0.7"
include_dir = "0.7"
log = "0.4"
mailparse = "0.15"
native-tls = "0.2"

4
fix-errors.sh
View file

@ -24,7 +24,7 @@ dirs=(
#"bot"
"bootstrap"
#"channels"
#"config"
"config"
#"context"
#"email"
#"file"
@ -53,7 +53,7 @@ cat "$PROJECT_ROOT/src/main.rs" >> "$OUTPUT_FILE"
echo "" >> "$OUTPUT_FILE"
echo "Compiling..."
echo "Compiling..."
cargo build --message-format=short 2>&1 | grep -E 'error' >> "$OUTPUT_FILE"

4
gbot.sh
View file

@ -1,2 +1,2 @@
clear && \
RUST_LOG=trace,hyper_util=off cargo build && clear && cargo run
pkill postgres && rm -rf botserver-stack && clear && \
RUST_LOG=trace,hyper_util=off cargo run

5
prompts/dev/platform/shared.md
View file

@ -1,4 +1,6 @@
MOST IMPORTANT CODE GENERATION RULES:
- Use rustc 1.90.0 (1159e78c4 2025-09-14).
- Check for warnings related to use of mut where is dispensable.
- No placeholders, never comment/uncomment code, no explanations, no filler text.
- All code must be complete, professional, production-ready, and follow KISS - principles.
- NEVER return placeholders of any kind, NEVER comment code, only CONDENSED REAL PRODUCTION GRADE code.
@ -14,7 +16,8 @@ MOST IMPORTANT CODE GENERATION RULES:
- Return *only the modified* files as a single `.sh` script using `cat`, so the code can be - restored directly.
- Pay attention to shared::utils and shared::models to reuse shared things.
- NEVER return a untouched file in output. Just files that need to be updated.
- Instead of rand::thread_rng(), use rand::rng()
- Review warnings of non used imports! Give me 0 warnings, please.
- You MUST return exactly this example format:
```sh
#!/bin/bash

87
src/bootstrap/mod.rs
View file

@ -1,9 +1,9 @@
use crate::config::AppConfig;
use crate::package_manager::{InstallMode, PackageManager};
use anyhow::Result;
use log::{trace, warn};
use diesel::{Connection, RunQueryDsl};
use log::trace;
use rand::distr::Alphanumeric;
use rand::rngs::ThreadRng;
use rand::Rng;
use sha2::{Digest, Sha256};
@ -63,7 +63,8 @@ impl BootstrapManager {
pub fn bootstrap(&mut self) -> Result<AppConfig> {
let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;
let required_components = vec!["tables"]; // , "cache", "drive", "llm"];
let required_components = vec!["tables"];
let mut config = AppConfig::from_env();
for component in required_components {
if !pm.is_installed(component) {
@ -71,46 +72,68 @@ impl BootstrapManager {
futures::executor::block_on(pm.install(component))?;
trace!("Starting component after install: {}", component);
pm.start(component)?;
if component == "tables" {
let db_password = self.generate_secure_password(16);
let farm_password = self.generate_secure_password(32);
let encrypted_db_password = self.encrypt_password(&db_password, &farm_password);
let env_contents = format!(
"FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver",
farm_password, db_password
);
std::fs::write(".env", env_contents)
.map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?;
trace!("Waiting 5 seconds for database to start...");
std::thread::sleep(std::time::Duration::from_secs(5));
let migration_dir = include_dir::include_dir!("./migrations");
let mut migration_files: Vec<_> = migration_dir
.files()
.filter_map(|file| {
let path = file.path();
if path.extension()? == "sql" {
Some(path.to_path_buf())
} else {
None
}
})
.collect();
migration_files.sort();
let mut conn = diesel::PgConnection::establish(&format!(
"postgres://gbuser:{}@localhost:5432/botserver",
db_password
))?;
for migration_file in migration_files {
let migration = std::fs::read_to_string(&migration_file)?;
trace!("Executing migration: {}", migration_file.display());
diesel::sql_query(&migration).execute(&mut conn)?;
}
self.setup_secure_credentials(&mut conn, &encrypted_db_password)?;
config = AppConfig::from_database(&mut conn);
}
} else {
trace!("Required component {} already installed", component);
}
}
let config = match diesel::Connection::establish(
"postgres://botserver:botserver@localhost:5432/botserver",
) {
Ok(mut conn) => {
self.setup_secure_credentials(&mut conn)?;
AppConfig::from_database(&mut conn)
}
Err(e) => {
warn!("Failed to connect to database for config: {}", e);
AppConfig::from_env()
}
};
Ok(config)
}
fn setup_secure_credentials(&self, conn: &mut diesel::PgConnection) -> Result<()> {
fn setup_secure_credentials(
&self,
conn: &mut diesel::PgConnection,
encrypted_db_password: &str,
) -> Result<()> {
use crate::shared::models::schema::bots::dsl::*;
use diesel::prelude::*;
use uuid::Uuid;
let farm_password =
std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32));
let db_password = self.generate_secure_password(16);
let encrypted_db_password = self.encrypt_password(&db_password, &farm_password);
let env_contents = format!(
"FARM_PASSWORD={}\nDATABASE_URL=postgres://gbuser:{}@localhost:5432/botserver",
farm_password, db_password
);
std::fs::write(".env", env_contents)
.map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?;
let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?;
diesel::update(bots)
.filter(bot_id.eq(system_bot_id))
@ -123,7 +146,7 @@ impl BootstrapManager {
}
fn generate_secure_password(&self, length: usize) -> String {
let rng: ThreadRng = rand::rng();
let rng = rand::rng();
rng.sample_iter(&Alphanumeric)
.take(length)
.map(char::from)

32
src/package_manager/facade.rs
View file

@ -132,7 +132,10 @@ impl PackageManager {
if !packages.is_empty() {
let pkg_list = packages.join(" ");
self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?;
self.exec_in_container(
&container_name,
&format!("apt-get update && apt-get install -y {}", pkg_list),
)?;
}
if let Some(url) = &component.download_url {
@ -266,10 +269,8 @@ impl PackageManager {
match self.os_type {
OsType::Linux => {
let output = Command::new("apt-get")
.args(&["update"])
.output()?;
let output = Command::new("apt-get").args(&["update"]).output()?;
if !output.status.success() {
warn!("apt-get update had issues");
}
@ -569,6 +570,7 @@ impl PackageManager {
.replace("{{LOGS_PATH}}", &logs_path.to_string_lossy());
if target == "local" {
trace!("Executing command: {}", rendered_cmd);
let output = Command::new("bash")
.current_dir(&bin_path)
.args(&["-c", &rendered_cmd])
@ -750,17 +752,17 @@ impl PackageManager {
])
.output()?;
if !output.status.success() {
warn!("Failed to setup port forwarding for port {}", port);
if !output.status.success() {
warn!("Failed to setup port forwarding for port {}", port);
}
trace!(
"Port forwarding configured: {} -> container {}",
port,
container
);
}
trace!(
"Port forwarding configured: {} -> container {}",
port,
container
);
Ok(())
}
Ok(())
}
}

17
src/package_manager/installer.rs
View file

@ -4,7 +4,6 @@ use crate::package_manager::{InstallMode, OsType};
use anyhow::Result;
use log::trace;
use rand::distr::Alphanumeric;
use rand::rngs::ThreadRng;
use rand::Rng;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@ -64,8 +63,7 @@ impl PackageManager {
fn register_drive(&mut self) {
let drive_password = self.generate_secure_password(16);
let farm_password =
std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32));
let farm_password = std::env::var("FARM_PASSWORD").unwrap();
let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password);
self.components.insert("drive".to_string(), ComponentConfig {
@ -110,10 +108,7 @@ impl PackageManager {
use diesel::pg::PgConnection;
use diesel::prelude::*;
use uuid::Uuid;
if let Ok(mut conn) =
PgConnection::establish("postgres://botserver:botserver@localhost:5432/botserver")
{
if let Ok(mut conn) = PgConnection::establish(&std::env::var("DATABASE_URL")?) {
let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?;
diesel::update(bots)
.filter(bot_id.eq(system_bot_id))
@ -148,7 +143,7 @@ impl PackageManager {
"if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"log_directory = '{{LOGS_PATH}}'\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(),
"if [ ! -f \"{{CONF_PATH}}/postgresql.conf\" ]; then echo \"logging_collector = on\" >> {{CONF_PATH}}/postgresql.conf; fi".to_string(),
"if [ ! -f \"{{CONF_PATH}}/pg_hba.conf\" ]; then echo \"host all all all md5\" > {{CONF_PATH}}/pg_hba.conf; fi".to_string(),
"if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi".to_string(),
"if [ ! -f \"{{CONF_PATH}}/pg_ident.conf\" ]; then touch {{CONF_PATH}}/pg_ident.conf; fi ".to_string(),
"if [ ! -d \"{{DATA_PATH}}/pgdata\" ]; then ./bin/pg_ctl -D {{DATA_PATH}}/pgdata -l {{LOGS_PATH}}/postgres.log start; sleep 5; ./bin/createdb -p 5432 -h localhost botserver; ./bin/createuser -p 5432 -h localhost gbuser; fi".to_string()
],
pre_install_cmds_macos: vec![],
@ -193,7 +188,7 @@ impl PackageManager {
self.components.insert("llm".to_string(), ComponentConfig {
name: "llm".to_string(),
required: true,
ports: vec![8081],
ports: vec![8081, 8082],
dependencies: vec![],
linux_packages: vec!["unzip".to_string()],
macos_packages: vec!["unzip".to_string()],
@ -213,7 +208,7 @@ impl PackageManager {
pre_install_cmds_windows: vec![],
post_install_cmds_windows: vec![],
env_vars: HashMap::new(),
exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081".to_string(),
exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081 & {{BIN_PATH}}/llama-server -m {{DATA_PATH}}/bge-small-en-v1.5-f32.gguf --port 8082 --embedding".to_string(),
});
}
@ -656,7 +651,7 @@ impl PackageManager {
}
fn generate_secure_password(&self, length: usize) -> String {
let rng: ThreadRng = rand::rng();
let rng = rand::rng();
rng.sample_iter(&Alphanumeric)
.take(length)
.map(char::from)