GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests Projects Releases 323 Packages Wiki Activity Actions

- Fine tunning GTP OSS 20B.

Browse source
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-10-14 16:57:50 -03:00
parent 573437cb6f
commit 277f21ab18
2 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
scripts/containers/system.sh
View file

@ -33,6 +33,7 @@ mv build/bin/* .
rm build/bin -r rm build/bin -r
rm llama-b6148-bin-ubuntu-x64.zip rm llama-b6148-bin-ubuntu-x64.zip
sudo apt install lib-pq
wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf wget https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-1.5B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-1.5B-Q3_K_M.gguf
wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf wget https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-f32.gguf

2
src/llm_legacy/llm_local.rs
View file

@ -192,7 +192,7 @@ async fn start_llm_server(
let mut cmd = tokio::process::Command::new("sh"); let mut cmd = tokio::process::Command::new("sh");
cmd.arg("-c").arg(format!( cmd.arg("-c").arg(format!(
"cd {} && ./llama-server -m {} --host 0.0.0.0 --port {} --n-gpu-layers 99 &", "cd {} && ./llama-server -m {} --host 0.0.0.0 --port {} {} --top_p 0.95 --temp 0.6 --flash-attn on --ctx-size 4096 --repeat-penalty 1.2 -ngl 22 &",
llama_cpp_path, model_path, port llama_cpp_path, model_path, port
)); ));