use as many threads as possible

.gitignore

@@ -91,3 +91,6 @@ typings/
 
 # Electron-Forge
 out/
+
+# yarn lock
+yarn.lock

package.json

@@ -66,6 +66,7 @@
     "cors": "^2.8.5",
     "electron-squirrel-startup": "^1.0.0",
     "express": "^4.18.2",
+    "os-utils": "^0.0.14",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-markdown": "^8.0.6",

src/index.ts

@@ -374,16 +374,26 @@ const checkIfFileExists = async () => {
   }
 };
 
+const osUtil = require("os-utils");
+var threads = 0;
+var sysThreads = osUtil.cpuCount();
+for (let i = 1; i < sysThreads; i = i * 2) {
+	threads = i;
+}
+if (sysThreads == 4) {
+	threads = 4;
+}
+
 io.on("connection", (socket) => {
   /* 
     The alpaca model doesnot work with context so we need to spawn a new process for each chat
     This is not ideal but it works for now. If you have any suggestions on how to improve this
     please let me know!
   */
-  let program = spawn(CHAT_APP_LOCATION, ["-m", FILEPATH]);
+  let program = spawn(CHAT_APP_LOCATION, ["-m", FILEPATH, "--threads", threads.toString(), "--batch_size", (threads * 4).toString()]);
 
   socket.on("chatstart", () => {
-    program = spawn(CHAT_APP_LOCATION, ["-m", FILEPATH]);
+    program = spawn(CHAT_APP_LOCATION, ["-m", FILEPATH, "--threads", threads.toString(), "--batch_size", (threads * 4).toString()]);
   });
 
   socket.on("stopResponding", () => {