From 3151dbc3f5cf3bac82e8a090d8148ed5e6140127 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Sun, 11 Aug 2024 00:51:47 +0900 Subject: [PATCH 1/9] Add .gitignore into root project directory(#279) --- .gitignore | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b1d25daf --- /dev/null +++ b/.gitignore @@ -0,0 +1,254 @@ +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,node,macos,windows,react +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,node,macos,windows,react + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local +.env.sentry-build-plugin + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist +dist-ssr + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + +### react ### +.DS_* +**/*.backup.* +**/*.back.* + +node_modules + +*.sublime* + +psd +thumb +sketch + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Sqlite +dev.db +dev.db-journal + +# Editor and files +.project +.classpath +.c9/ +*.launch +.settings/ +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +.idea/ + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,node,macos,windows,react From a21e55c7baa94894b03a5f7a1096778962003cf7 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Mon, 12 Aug 2024 21:52:44 +0900 Subject: [PATCH 2/9] chore: just combine front, back ignore file - remove .gitignore in each folder --- .gitignore | 250 ++++---------------------------------------- backend/.gitignore | 42 -------- frontend/.gitignore | 29 ----- 3 files changed, 22 insertions(+), 299 deletions(-) delete mode 100644 backend/.gitignore delete mode 100755 frontend/.gitignore diff --git a/.gitignore b/.gitignore index b1d25daf..58ad8aa2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,254 +1,48 @@ -# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,node,macos,windows,react -# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,node,macos,windows,react - -### macOS ### -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### macOS Patch ### -# iCloud generated files -*.icloud - -### Node ### # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* +pnpm-debug.log* lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local -.env.sentry-build-plugin - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist -dist-ssr - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp - -# Docusaurus cache and generated files -.docusaurus +dist/ +dist-ssr/ +*.local -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -### Node Patch ### -# Serverless Webpack directories -.webpack/ - -# Optional stylelint cache - -# SvelteKit build / generate output -.svelte-kit - -### react ### -.DS_* -**/*.backup.* -**/*.back.* - -node_modules - -*.sublime* - -psd -thumb -sketch - -### VisualStudioCode ### +# Editor directories and files .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json -!.vscode/*.code-snippets - -# Local History for Visual Studio Code -.history/ - -# Built Visual Studio Code Extensions -*.vsix - -### VisualStudioCode Patch ### -# Ignore all local history of files -.history -.ionide - -### Windows ### -# Windows thumbnail cache files -Thumbs.db -Thumbs.db:encryptable -ehthumbs.db -ehthumbs_vista.db - -# Dump file -*.stackdump - -# Folder config file -[Dd]esktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msix -*.msm -*.msp - -# Windows shortcuts -*.lnk - -# Sqlite -dev.db -dev.db-journal - -# Editor and files +.DS_Store +.idea .project .classpath .c9/ *.launch .settings/ +*.sublime-workspace *.suo *.ntvs* *.njsproj *.sln *.sw? -.idea/ -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,node,macos,windows,react +# Environment +.env +.env.development +.env.sentry-build-plugin + + +# Tests +coverage/ +.nyc_output/ + + +# Sqlite +dev.db +dev.db-journal diff --git a/backend/.gitignore b/backend/.gitignore deleted file mode 100644 index c630a236..00000000 --- a/backend/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -# compiled output -/dist -/node_modules - -# Logs -logs -*.log -npm-debug.log* -pnpm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# OS -.DS_Store - -# Tests -/coverage -/.nyc_output - -# IDEs and editors -/.idea -.project -.classpath -.c9/ -*.launch -.settings/ -*.sublime-workspace - -# IDE - VSCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json - -# Sqlite -dev.db -dev.db-journal - -# env -.env \ No newline at end of file diff --git a/frontend/.gitignore b/frontend/.gitignore deleted file mode 100755 index 1e6edccd..00000000 --- a/frontend/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? - -# Environment -.env -.env.development -.env.sentry-build-plugin From 9ec2ba6f7f625c99958789df374cfa6c5585a2e8 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Tue, 13 Aug 2024 09:44:48 +0900 Subject: [PATCH 3/9] chore: fix legacy file & seperate OD part in gitignore --- .gitignore | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 58ad8aa2..683ef965 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ dist-ssr/ !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json -.DS_Store .idea .project .classpath @@ -32,6 +31,10 @@ dist-ssr/ *.sln *.sw? +# OS +.DS_Store + + # Environment .env .env.development @@ -41,8 +44,3 @@ dist-ssr/ # Tests coverage/ .nyc_output/ - - -# Sqlite -dev.db -dev.db-journal From aadd96ccf727a52aae0e28eea3bb9e1f0d61d452 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Thu, 15 Aug 2024 01:47:26 +0900 Subject: [PATCH 4/9] Add ollama llm for yorkie intelligence # 255 - add docker image in docker compose file - change yorkie intelligence env var - add lib related to ollama --- backend/.env.development | 9 +- backend/docker/docker-compose-full.yml | 6 + backend/docker/docker-compose.yml | 6 + backend/package-lock.json | 159 ++++++++++++++++++---- backend/package.json | 1 + backend/src/langchain/langchain.module.ts | 15 +- backend/src/settings/settings.service.ts | 2 +- 7 files changed, 167 insertions(+), 31 deletions(-) diff --git a/backend/.env.development b/backend/.env.development index 236e5076..aa5e2bc4 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -5,10 +5,10 @@ DATABASE_URL=mongodb://localhost:27017/codepair # GITHUB_CLIENT_ID: Client ID for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_ID=your_github_client_id_here +GITHUB_CLIENT_ID="" # GITHUB_CLIENT_SECRET: Client secret for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_SECRET=your_github_client_secret_here +GITHUB_CLIENT_SECRET="" # GITHUB_CALLBACK_URL: Callback URL for handling GitHub authentication response. # Format: https:///auth/login/github # Example: http://localhost:3000/auth/login/github (For development mode) @@ -37,9 +37,10 @@ YORKIE_PROJECT_NAME=default YORKIE_PROJECT_SECRET_KEY="" # YORKIE_INTELLIGENCE: Whether to enable Yorkie Intelligence for collaborative editing. -# Set to true if Yorkie Intelligence is required. +# Set to ollama modelname if Yorkie Intelligence is required +# you can find llm model in https://ollama.com/library # If set to false, OPENAI_API_KEY is not required. -YORKIE_INTELLIGENCE=false +YORKIE_INTELLIGENCE="gemma2:2b" # OPENAI_API_KEY: API key for using the gpt-3.5-turbo model by Yorkie Intelligence. # To obtain an API key, visit OpenAI: https://help.openai.com/en/articles/4936850-where-do-i-find-my-api-key OPENAI_API_KEY=your_openai_api_key_here diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index 5bec6f56..816bf7e4 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -40,6 +40,12 @@ services: - "8080:8080" - "8081:8081" + yorkie-intelligence: + image: "ollama/ollama:latest" + restart: always + ports: + - "11434:11434" + mongo: build: context: ./mongodb_replica diff --git a/backend/docker/docker-compose.yml b/backend/docker/docker-compose.yml index 7b191618..018df278 100644 --- a/backend/docker/docker-compose.yml +++ b/backend/docker/docker-compose.yml @@ -9,6 +9,12 @@ services: - "8080:8080" - "8081:8081" + yorkie-intelligence: + image: "ollama/ollama:latest" + restart: always + ports: + - "11434:11434" + mongo: build: context: ./mongodb_replica diff --git a/backend/package-lock.json b/backend/package-lock.json index 806b84aa..1ae71216 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/s3-request-presigner": "^3.509.0", "@langchain/community": "^0.0.21", "@langchain/core": "^0.1.18", + "@langchain/ollama": "^0.0.4", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.0.0", @@ -2999,6 +3000,107 @@ "node": ">=18" } }, + "node_modules/@langchain/ollama": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.0.4.tgz", + "integrity": "sha512-laYaYFZsbu0Mjhm40CypXPV5gYvQjd16oBEoGnFFjmCUgCCu6gjETg3tgvgifo7w+dhmNPLUyqnjXA/BpWEH2Q==", + "license": "MIT", + "dependencies": { + "@langchain/core": ">=0.2.21 <0.3.0", + "ollama": "^0.5.6", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/ollama/node_modules/@langchain/core": { + "version": "0.2.23", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.23.tgz", + "integrity": "sha512-elPg6WpAkxWEIGC9u38F2anbzqfYYEy32lJdsd9dtChcHSFmFLlXqa+SnpO3R772gUuJmcu+Pd+fCvmRFy029w==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "~0.1.39", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/ollama/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@langchain/ollama/node_modules/langsmith": { + "version": "0.1.41", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.41.tgz", + "integrity": "sha512-8R7s/225Pxmv0ipMfd6sqmWVsfHLQivYlQZ0vx5K+ReoknummTenQlVK8gapk3kqRMnzkrouuRHMhWjMR6RgUA==", + "license": "MIT", + "dependencies": { + "@types/uuid": "^9.0.1", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^9.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/@langchain/ollama/node_modules/langsmith/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@langchain/ollama/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@langchain/openai": { "version": "0.0.13", "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.0.13.tgz", @@ -9293,9 +9395,10 @@ } }, "node_modules/js-tiktoken": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.10.tgz", - "integrity": "sha512-ZoSxbGjvGyMT13x6ACo9ebhDha/0FHdKA+OsQcMOWcm1Zs7r90Rhk5lhERLzji+3rA7EKpXCgwXcM5fF3DMpdA==", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.12.tgz", + "integrity": "sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==", + "license": "MIT", "dependencies": { "base64-js": "^1.5.1" } @@ -10387,6 +10490,15 @@ "node": ">= 6.0.0" } }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -10542,6 +10654,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ollama": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.8.tgz", + "integrity": "sha512-frBGdfSV34i7JybLZUeyCYDx0CMyDiG4On8xOK+cNRWM04HImhoWgIMpF4p7vTkQumadbSxOteR7SZyKqNmOXg==", + "license": "MIT", + "dependencies": { + "whatwg-fetch": "^3.6.20" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -11731,12 +11852,10 @@ "dev": true }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -11744,22 +11863,6 @@ "node": ">=10" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/send": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", @@ -13178,6 +13281,12 @@ "node": ">=4.0" } }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", + "license": "MIT" + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", diff --git a/backend/package.json b/backend/package.json index bf663664..d40ac36a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -32,6 +32,7 @@ "@aws-sdk/s3-request-presigner": "^3.509.0", "@langchain/community": "^0.0.21", "@langchain/core": "^0.1.18", + "@langchain/ollama": "^0.0.4", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.0.0", diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index 73771480..fb72abfb 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -1,10 +1,23 @@ import { Module } from "@nestjs/common"; import { ChatOpenAI } from "@langchain/openai"; +import { ChatOllama } from "@langchain/ollama"; import { BaseChatModel } from "@langchain/core/language_models/chat_models"; const chatModelFactory = { provide: "ChatModel", - useFactory: () => new ChatOpenAI({ modelName: "gpt-4o-mini" }) as BaseChatModel, + useFactory: () => { + const modelType = process.env.YORKIE_INTELLIGENCE; + if (modelType === "gemma2:2b"){ + return new ChatOllama({ + model: modelType, + checkOrPullModel: true, + streaming: true, + }); + } + else if (modelType === "openai"){ + return new ChatOpenAI({ modelName: "gpt-4o-mini" }) as BaseChatModel; + } + } }; @Module({ diff --git a/backend/src/settings/settings.service.ts b/backend/src/settings/settings.service.ts index a2532705..c387c3a7 100644 --- a/backend/src/settings/settings.service.ts +++ b/backend/src/settings/settings.service.ts @@ -10,7 +10,7 @@ export class SettingsService { async getSettings(): Promise { return { yorkieIntelligence: { - enable: this.configService.get("YORKIE_INTELLIGENCE") === "true", + enable: this.configService.get("YORKIE_INTELLIGENCE") !== "false", config: { features: generateFeatureList(this.configService), }, From 2dadca1f506e634fe9bbdf8b6f191dd3b7fc5c06 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Thu, 15 Aug 2024 12:16:48 +0900 Subject: [PATCH 5/9] apply formatting --- backend/src/langchain/langchain.module.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index fb72abfb..00e35da4 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -7,17 +7,16 @@ const chatModelFactory = { provide: "ChatModel", useFactory: () => { const modelType = process.env.YORKIE_INTELLIGENCE; - if (modelType === "gemma2:2b"){ + if (modelType === "gemma2:2b") { return new ChatOllama({ model: modelType, checkOrPullModel: true, streaming: true, }); - } - else if (modelType === "openai"){ + } else if (modelType === "openai") { return new ChatOpenAI({ modelName: "gpt-4o-mini" }) as BaseChatModel; } - } + }, }; @Module({ From 0d2d77a1d394bc9a2848a34a2a19e9a6409b9647 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Sun, 18 Aug 2024 15:28:41 +0900 Subject: [PATCH 6/9] Add ollama model option #255 - fix docker compose file (user can change ollama conatainer port) - fix readme docs(add --env-file option) - add usable model --- README.md | 4 ++-- backend/.env.development | 8 ++++++-- backend/README.md | 2 +- backend/docker/docker-compose-full.yml | 2 +- backend/docker/docker-compose.yml | 2 +- backend/src/langchain/langchain.module.ts | 13 ++++++++++++- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ae6642d2..62f4c76d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ We offer two options. Choose the one that best suits your needs: 2. Run `./backend/docker/docker-compose-full.yml`. ```bash - docker-compose -f ./backend/docker/docker-compose-full.yml up -d + docker-compose -f ./backend/docker/docker-compose-full.yml --env-file ./backend/.env.development up -d ``` 3. Run the Frontend application: @@ -88,7 +88,7 @@ We offer two options. Choose the one that best suits your needs: 2. Run `./backend/docker/docker-compose.yml`. ```bash - docker-compose -f ./backend/docker/docker-compose.yml up -d + docker-compose -f ./backend/docker/docker-compose.yml --env-file ./backend/.env.development up -d ``` 3. Run the Backend application: diff --git a/backend/.env.development b/backend/.env.development index aa5e2bc4..fb7ed73d 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -5,10 +5,10 @@ DATABASE_URL=mongodb://localhost:27017/codepair # GITHUB_CLIENT_ID: Client ID for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_ID="" +GITHUB_CLIENT_ID=your_github_client_id_here # GITHUB_CLIENT_SECRET: Client secret for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_SECRET="" +GITHUB_CLIENT_SECRET=your_github_client_secret_here # GITHUB_CALLBACK_URL: Callback URL for handling GitHub authentication response. # Format: https:///auth/login/github # Example: http://localhost:3000/auth/login/github (For development mode) @@ -41,6 +41,10 @@ YORKIE_PROJECT_SECRET_KEY="" # you can find llm model in https://ollama.com/library # If set to false, OPENAI_API_KEY is not required. YORKIE_INTELLIGENCE="gemma2:2b" + +# OLLAMA_HOST_PORT: yorkie-intelligence conatiner port +OLLAMA_HOST_PORT=11434 + # OPENAI_API_KEY: API key for using the gpt-3.5-turbo model by Yorkie Intelligence. # To obtain an API key, visit OpenAI: https://help.openai.com/en/articles/4936850-where-do-i-find-my-api-key OPENAI_API_KEY=your_openai_api_key_here diff --git a/backend/README.md b/backend/README.md index 9db6d20d..2b57266e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -24,7 +24,7 @@ This project is the backend part of the CodePair service developed using NestJS. 3. Run `.backend/docker/docker-compose.yml`. ```bash - docker-compose -f ./backend/docker/docker-compose.yml up -d + docker-compose -f ./backend/docker/docker-compose.yml --env-file ./backend/.env.development up -d ``` 4. Run the Backend application: diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index 816bf7e4..4596518c 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -44,7 +44,7 @@ services: image: "ollama/ollama:latest" restart: always ports: - - "11434:11434" + - ${OLLAMA_HOST_PORT}:11434 mongo: build: diff --git a/backend/docker/docker-compose.yml b/backend/docker/docker-compose.yml index 018df278..667a33d0 100644 --- a/backend/docker/docker-compose.yml +++ b/backend/docker/docker-compose.yml @@ -13,7 +13,7 @@ services: image: "ollama/ollama:latest" restart: always ports: - - "11434:11434" + - ${OLLAMA_HOST_PORT}:11434 mongo: build: diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index 00e35da4..b047c9a7 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -6,10 +6,21 @@ import { BaseChatModel } from "@langchain/core/language_models/chat_models"; const chatModelFactory = { provide: "ChatModel", useFactory: () => { + let modelList: string[] = [ + "lamma3.1", + "gemma2", + "gemma2:2b", + "phi3", + "mistral", + "neural-chat", + "starling-lm", + "solar", + ]; const modelType = process.env.YORKIE_INTELLIGENCE; - if (modelType === "gemma2:2b") { + if (modelType in modelList) { return new ChatOllama({ model: modelType, + baseUrl: process.env.OLLAMA_HOST_PORT, checkOrPullModel: true, streaming: true, }); From 0ad41b1a524689c74172d972349a74b5eb766cfd Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Tue, 20 Aug 2024 20:40:09 +0900 Subject: [PATCH 7/9] feat: add modelList type, change port to baseurl #255 - apply github code review --- backend/.env.development | 10 +++-- backend/docker/docker-compose-full.yml | 4 +- backend/docker/docker-compose.yml | 4 +- backend/src/langchain/langchain.module.ts | 48 +++++++++++++++++++---- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/backend/.env.development b/backend/.env.development index aa5e2bc4..0e67ee63 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -5,10 +5,10 @@ DATABASE_URL=mongodb://localhost:27017/codepair # GITHUB_CLIENT_ID: Client ID for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_ID="" +GITHUB_CLIENT_ID=your_github_client_id_here # GITHUB_CLIENT_SECRET: Client secret for authenticating with GitHub. # To obtain a client ID, create an OAuth app at: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app -GITHUB_CLIENT_SECRET="" +GITHUB_CLIENT_SECRET=your_github_client_secret_here # GITHUB_CALLBACK_URL: Callback URL for handling GitHub authentication response. # Format: https:///auth/login/github # Example: http://localhost:3000/auth/login/github (For development mode) @@ -40,7 +40,11 @@ YORKIE_PROJECT_SECRET_KEY="" # Set to ollama modelname if Yorkie Intelligence is required # you can find llm model in https://ollama.com/library # If set to false, OPENAI_API_KEY is not required. -YORKIE_INTELLIGENCE="gemma2:2b" +YORKIE_INTELLIGENCE="ollama:gemma2:2b" + +# OLLAMA_HOST_URL: yorkie-intelligence ollama url +OLLAMA_HOST_URL=http://localhost:11434 + # OPENAI_API_KEY: API key for using the gpt-3.5-turbo model by Yorkie Intelligence. # To obtain an API key, visit OpenAI: https://help.openai.com/en/articles/4936850-where-do-i-find-my-api-key OPENAI_API_KEY=your_openai_api_key_here diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index 816bf7e4..2fddd53f 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -43,8 +43,8 @@ services: yorkie-intelligence: image: "ollama/ollama:latest" restart: always - ports: - - "11434:11434" + expose: + - "11434" mongo: build: diff --git a/backend/docker/docker-compose.yml b/backend/docker/docker-compose.yml index 018df278..a4845797 100644 --- a/backend/docker/docker-compose.yml +++ b/backend/docker/docker-compose.yml @@ -12,8 +12,8 @@ services: yorkie-intelligence: image: "ollama/ollama:latest" restart: always - ports: - - "11434:11434" + expose: + - "11434" mongo: build: diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index 00e35da4..f7b8e449 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -3,18 +3,50 @@ import { ChatOpenAI } from "@langchain/openai"; import { ChatOllama } from "@langchain/ollama"; import { BaseChatModel } from "@langchain/core/language_models/chat_models"; +type ModelList = { + [key: string]: string[]; +}; + +const modelList: ModelList = { + ollama: [ + "lamma3.1", + "gemma2", + "gemma2:2b", + "phi3", + "mistral", + "neural-chat", + "starling-lm", + "solar", + ], + openai: ["gpt-3.5-turbo", "gpt-4o-mini"], +}; + const chatModelFactory = { provide: "ChatModel", useFactory: () => { const modelType = process.env.YORKIE_INTELLIGENCE; - if (modelType === "gemma2:2b") { - return new ChatOllama({ - model: modelType, - checkOrPullModel: true, - streaming: true, - }); - } else if (modelType === "openai") { - return new ChatOpenAI({ modelName: "gpt-4o-mini" }) as BaseChatModel; + try { + const [provider, model] = modelType.split(":", 2); + let chatModel: BaseChatModel | ChatOllama; + + if (modelList[provider] && modelList[provider].includes(model)) { + if (provider === "ollama") { + chatModel = new ChatOllama({ + model: model, + baseUrl: process.env.OLLAMA_HOST_URL, + checkOrPullModel: true, + streaming: true, + }); + } else if (provider === "openai") { + chatModel = new ChatOpenAI({ modelName: model }); + } + } + + if (!chatModel) throw new Error(`${model} is not found. please check your model name`); + + return chatModel; + } catch { + throw new Error("~~~"); } }, }; From 0674eeab806aaa24f42e878c9ff1c48b21b48126 Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Tue, 20 Aug 2024 20:58:05 +0900 Subject: [PATCH 8/9] fix: apply npm format --- backend/src/langchain/langchain.module.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index 2d42062a..6b1bece1 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -5,7 +5,7 @@ import { BaseChatModel } from "@langchain/core/language_models/chat_models"; type ModelList = { [key: string]: string[]; -} +}; const modelList: ModelList = { ollama: [ @@ -18,22 +18,18 @@ const modelList: ModelList = { "starling-lm", "solar", ], - openai: [ - "gpt-3.5-turbo", - "gpt-4o-mini" - ] -} + openai: ["gpt-3.5-turbo", "gpt-4o-mini"], +}; const chatModelFactory = { provide: "ChatModel", useFactory: () => { - const modelType = process.env.YORKIE_INTELLIGENCE; - try{ + try { const [provider, model] = modelType.split(":", 2); - let chatModel: BaseChatModel | ChatOllama + let chatModel: BaseChatModel | ChatOllama; - if (modelList[provider] && modelList[provider].includes(model)){ + if (modelList[provider] && modelList[provider].includes(model)) { if (provider === "ollama") { chatModel = new ChatOllama({ model: model, @@ -52,7 +48,6 @@ const chatModelFactory = { } catch { throw new Error(`${modelType} is not found. please check your model name`); } - }, }; From 4ec2f73990b8e53d1dcd385528f583b29ee9682c Mon Sep 17 00:00:00 2001 From: sihyeong671 Date: Tue, 20 Aug 2024 22:11:57 +0900 Subject: [PATCH 9/9] fix: refactor by github review --- backend/src/langchain/langchain.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/langchain/langchain.module.ts b/backend/src/langchain/langchain.module.ts index 6b1bece1..1a9e49b0 100644 --- a/backend/src/langchain/langchain.module.ts +++ b/backend/src/langchain/langchain.module.ts @@ -29,7 +29,7 @@ const chatModelFactory = { const [provider, model] = modelType.split(":", 2); let chatModel: BaseChatModel | ChatOllama; - if (modelList[provider] && modelList[provider].includes(model)) { + if (modelList[provider]?.includes(model)) { if (provider === "ollama") { chatModel = new ChatOllama({ model: model,