From 80b035008bae92032496aa8530fca36099290487 Mon Sep 17 00:00:00 2001
From: Louis <louis@jan.ai>
Date: Mon, 12 Aug 2024 17:06:06 +0700
Subject: [PATCH] chore: system resource events should include gpu system
 information (#1011)

---
 cortex-js/src/domain/models/resource.interface.ts      |  6 ++++++
 .../resources-manager/resources-manager.service.ts     | 10 ++++++----
 cortex-js/src/usecases/models/models.usecases.ts       |  2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/cortex-js/src/domain/models/resource.interface.ts b/cortex-js/src/domain/models/resource.interface.ts
index 72741d7f7..16dafb9dc 100644
--- a/cortex-js/src/domain/models/resource.interface.ts
+++ b/cortex-js/src/domain/models/resource.interface.ts
@@ -7,9 +7,15 @@ export interface ResourceStatus {
   cpu: {
     usage: number;
   };
+  gpus: GpuInfo[];
 }
 
 export interface UsedMemInfo {
   total: number;
   used: number;
 }
+
+export interface GpuInfo {
+  name: string | undefined;
+  vram: number | null;
+}
diff --git a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts
index 8b8752af6..565c788dc 100644
--- a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts
+++ b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts
@@ -1,16 +1,15 @@
-import osUtils from 'node-os-utils'
 import {
   ResourceStatus,
   UsedMemInfo,
 } from '@/domain/models/resource.interface';
 import { getMemoryInformation, MemoryInformation } from '@/utils/system-resource';
 import { Injectable } from '@nestjs/common';
-import systemInformation, { Systeminformation } from 'systeminformation';
+import si, { Systeminformation } from 'systeminformation';
 
 @Injectable()
 export class ResourcesManagerService {
   async getResourceStatuses(): Promise<ResourceStatus> {
-    const promises = [systemInformation.currentLoad(), getMemoryInformation()];
+    const promises = [si.currentLoad(), getMemoryInformation()];
     const results = await Promise.all(promises);
 
     const cpuUsage = results[0] as Systeminformation.CurrentLoadData;
@@ -19,12 +18,15 @@ export class ResourcesManagerService {
       total: memory.total,
       used: memory.used,
     };
-
     return {
       mem: memInfo,
       cpu: {
         usage: Number(cpuUsage.currentLoad.toFixed(2)),
       },
+      gpus: (await si.graphics()).controllers.map((gpu) => ({
+        name: gpu.name,
+        vram: gpu.vram,
+      })),
     };
   }
 }
diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts
index 7182436cf..260c966c1 100644
--- a/cortex-js/src/usecases/models/models.usecases.ts
+++ b/cortex-js/src/usecases/models/models.usecases.ts
@@ -513,7 +513,7 @@ export class ModelsUsecases {
 
       // Default Inference Params
       stream: true,
-      max_tokens: 4098,
+      max_tokens: 4096,
       frequency_penalty: 0.7,
       presence_penalty: 0.7,
       temperature: 0.7,