Add include_dir dependency and use it for embedded migrations

Use include_dir to embed migration scripts and load them at runtime.
This change allows for easier management and versioning of migrations.

Cargo.lock

@@ -1032,6 +1032,7 @@ dependencies = [
  "futures-util",
  "headless_chrome",
  "imap",
+ "include_dir",
  "indicatif",
  "lettre",
  "livekit",
@@ -2855,6 +2856,25 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
 
+[[package]]
+name = "include_dir"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
+dependencies = [
+ "include_dir_macros",
+]
+
+[[package]]
+name = "include_dir_macros"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"

Cargo.toml

@@ -63,6 +63,7 @@ futures = "0.3"
 futures-util = "0.3"
 lettre = { version = "0.11", features = ["smtp-transport", "builder", "tokio1", "tokio1-native-tls"] }
 livekit = "0.7"
+include_dir = "0.7"
 log = "0.4"
 mailparse = "0.15"
 native-tls = "0.2"

fix-errors.sh

@@ -24,7 +24,7 @@ dirs=(
     #"bot"
     "bootstrap"
     #"channels"
-    #"config"
+    "config"
     #"context"
     #"email"
     #"file"

gbot.sh

@@ -1,2 +1,2 @@
-clear && \
+pkill postgres && rm -rf botserver-stack && clear && \
-     RUST_LOG=trace,hyper_util=off cargo build && clear && cargo run 
+     RUST_LOG=trace,hyper_util=off cargo run 

prompts/dev/platform/shared.md

@@ -1,4 +1,6 @@
 MOST IMPORTANT CODE GENERATION RULES:
+- Use rustc 1.90.0 (1159e78c4 2025-09-14).
+- Check for warnings related to use of mut where is dispensable.
 - No placeholders, never comment/uncomment code, no explanations, no filler text.
 - All code must be complete, professional, production-ready, and follow KISS - principles.
 - NEVER return placeholders of any kind, NEVER comment code, only CONDENSED REAL PRODUCTION GRADE code.
@@ -14,7 +16,8 @@ MOST IMPORTANT CODE GENERATION RULES:
 - Return *only the modified* files as a single `.sh` script using `cat`, so the code can be - restored directly.
 - Pay attention to shared::utils and shared::models to reuse shared things.
 - NEVER return a untouched file in output. Just files that need to be updated.
-
+- Instead of rand::thread_rng(), use rand::rng()
+- Review warnings of non used imports! Give me 0 warnings, please.
 - You MUST return exactly this example format:
 ```sh
 #!/bin/bash

src/bootstrap/mod.rs

@@ -1,9 +1,9 @@
 use crate::config::AppConfig;
 use crate::package_manager::{InstallMode, PackageManager};
 use anyhow::Result;
-use log::{trace, warn};
+use diesel::{Connection, RunQueryDsl};
+use log::trace;
 use rand::distr::Alphanumeric;
-use rand::rngs::ThreadRng;
 use rand::Rng;
 use sha2::{Digest, Sha256};
 
@@ -63,7 +63,8 @@ impl BootstrapManager {
 
     pub fn bootstrap(&mut self) -> Result<AppConfig> {
         let pm = PackageManager::new(self.install_mode.clone(), self.tenant.clone())?;
-        let required_components = vec!["tables"]; // , "cache", "drive", "llm"];
+        let required_components = vec!["tables"];
+        let mut config = AppConfig::from_env();
 
         for component in required_components {
             if !pm.is_installed(component) {
@@ -71,36 +72,10 @@ impl BootstrapManager {
                 futures::executor::block_on(pm.install(component))?;
                 trace!("Starting component after install: {}", component);
                 pm.start(component)?;
-            } else {
-                trace!("Required component {} already installed", component);
-            }
-        }
 
-        let config = match diesel::Connection::establish(
+                if component == "tables" {
-            "postgres://botserver:botserver@localhost:5432/botserver",
-        ) {
-            Ok(mut conn) => {
-                self.setup_secure_credentials(&mut conn)?;
-                AppConfig::from_database(&mut conn)
-            }
-            Err(e) => {
-                warn!("Failed to connect to database for config: {}", e);
-                AppConfig::from_env()
-            }
-        };
-
-        Ok(config)
-    }
-
-    fn setup_secure_credentials(&self, conn: &mut diesel::PgConnection) -> Result<()> {
-        use crate::shared::models::schema::bots::dsl::*;
-        use diesel::prelude::*;
-        use uuid::Uuid;
-
-        let farm_password =
-            std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32));
                     let db_password = self.generate_secure_password(16);
-
+                    let farm_password = self.generate_secure_password(32);
                     let encrypted_db_password = self.encrypt_password(&db_password, &farm_password);
 
                     let env_contents = format!(
@@ -111,6 +86,54 @@ impl BootstrapManager {
                     std::fs::write(".env", env_contents)
                         .map_err(|e| anyhow::anyhow!("Failed to write .env file: {}", e))?;
 
+                    trace!("Waiting 5 seconds for database to start...");
+                    std::thread::sleep(std::time::Duration::from_secs(5));
+
+                    let migration_dir = include_dir::include_dir!("./migrations");
+                    let mut migration_files: Vec<_> = migration_dir
+                        .files()
+                        .filter_map(|file| {
+                            let path = file.path();
+                            if path.extension()? == "sql" {
+                                Some(path.to_path_buf())
+                            } else {
+                                None
+                            }
+                        })
+                        .collect();
+
+                    migration_files.sort();
+                    let mut conn = diesel::PgConnection::establish(&format!(
+                        "postgres://gbuser:{}@localhost:5432/botserver",
+                        db_password
+                    ))?;
+
+                    for migration_file in migration_files {
+                        let migration = std::fs::read_to_string(&migration_file)?;
+                        trace!("Executing migration: {}", migration_file.display());
+                        diesel::sql_query(&migration).execute(&mut conn)?;
+                    }
+
+                    self.setup_secure_credentials(&mut conn, &encrypted_db_password)?;
+                    config = AppConfig::from_database(&mut conn);
+                }
+            } else {
+                trace!("Required component {} already installed", component);
+            }
+        }
+
+        Ok(config)
+    }
+
+    fn setup_secure_credentials(
+        &self,
+        conn: &mut diesel::PgConnection,
+        encrypted_db_password: &str,
+    ) -> Result<()> {
+        use crate::shared::models::schema::bots::dsl::*;
+        use diesel::prelude::*;
+        use uuid::Uuid;
+
         let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?;
         diesel::update(bots)
             .filter(bot_id.eq(system_bot_id))
@@ -123,7 +146,7 @@ impl BootstrapManager {
     }
 
     fn generate_secure_password(&self, length: usize) -> String {
-        let rng: ThreadRng = rand::rng();
+        let rng = rand::rng();
         rng.sample_iter(&Alphanumeric)
             .take(length)
             .map(char::from)

src/package_manager/facade.rs

@@ -132,7 +132,10 @@ impl PackageManager {
 
         if !packages.is_empty() {
             let pkg_list = packages.join(" ");
-            self.exec_in_container(&container_name, &format!("apt-get update && apt-get install -y {}", pkg_list))?;
+            self.exec_in_container(
+                &container_name,
+                &format!("apt-get update && apt-get install -y {}", pkg_list),
+            )?;
         }
 
         if let Some(url) = &component.download_url {
@@ -266,9 +269,7 @@ impl PackageManager {
 
         match self.os_type {
             OsType::Linux => {
-                let output = Command::new("apt-get")
+                let output = Command::new("apt-get").args(&["update"]).output()?;
-                    .args(&["update"])
-                    .output()?;
 
                 if !output.status.success() {
                     warn!("apt-get update had issues");
@@ -569,6 +570,7 @@ impl PackageManager {
                 .replace("{{LOGS_PATH}}", &logs_path.to_string_lossy());
 
             if target == "local" {
+                trace!("Executing command: {}", rendered_cmd);
                 let output = Command::new("bash")
                     .current_dir(&bin_path)
                     .args(&["-c", &rendered_cmd])

src/package_manager/installer.rs

@@ -4,7 +4,6 @@ use crate::package_manager::{InstallMode, OsType};
 use anyhow::Result;
 use log::trace;
 use rand::distr::Alphanumeric;
-use rand::rngs::ThreadRng;
 use rand::Rng;
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
@@ -64,8 +63,7 @@ impl PackageManager {
 
     fn register_drive(&mut self) {
         let drive_password = self.generate_secure_password(16);
-        let farm_password =
+        let farm_password = std::env::var("FARM_PASSWORD").unwrap();
-            std::env::var("FARM_PASSWORD").unwrap_or_else(|_| self.generate_secure_password(32));
         let encrypted_drive_password = self.encrypt_password(&drive_password, &farm_password);
 
         self.components.insert("drive".to_string(), ComponentConfig {
@@ -110,10 +108,7 @@ impl PackageManager {
         use diesel::pg::PgConnection;
         use diesel::prelude::*;
         use uuid::Uuid;
-
+        if let Ok(mut conn) = PgConnection::establish(&std::env::var("DATABASE_URL")?) {
-        if let Ok(mut conn) =
-            PgConnection::establish("postgres://botserver:botserver@localhost:5432/botserver")
-        {
             let system_bot_id = Uuid::parse_str("00000000-0000-0000-0000-000000000000")?;
             diesel::update(bots)
                 .filter(bot_id.eq(system_bot_id))
@@ -193,7 +188,7 @@ impl PackageManager {
         self.components.insert("llm".to_string(), ComponentConfig {
             name: "llm".to_string(),
             required: true,
-            ports: vec![8081],
+            ports: vec![8081, 8082],
             dependencies: vec![],
             linux_packages: vec!["unzip".to_string()],
             macos_packages: vec!["unzip".to_string()],
@@ -213,7 +208,7 @@ impl PackageManager {
             pre_install_cmds_windows: vec![],
             post_install_cmds_windows: vec![],
             env_vars: HashMap::new(),
-            exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081".to_string(),
+            exec_cmd: "{{BIN_PATH}}/llama-server -m {{DATA_PATH}}/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf --port 8081 & {{BIN_PATH}}/llama-server -m {{DATA_PATH}}/bge-small-en-v1.5-f32.gguf --port 8082 --embedding".to_string(),
         });
     }
 
@@ -656,7 +651,7 @@ impl PackageManager {
     }
 
     fn generate_secure_password(&self, length: usize) -> String {
-        let rng: ThreadRng = rand::rng();
+        let rng = rand::rng();
         rng.sample_iter(&Alphanumeric)
             .take(length)
             .map(char::from)