From 3d07873da37cd09c4a6e6d1191e9f84c396b42fe Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Fri, 12 Jan 2024 10:38:16 +0800 Subject: [PATCH 1/4] feat(service): fix requirements.txt --- README_zh.md | 11 +++++++++-- requirements.txt | 7 +++++++ service/llm_server_hybrid.py | 3 ++- service/worker.py | 2 +- tests/test_hf_import_accelerate.py | 12 ++++++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/test_hf_import_accelerate.py diff --git a/README_zh.md b/README_zh.md index 4619bc45..22a47298 100644 --- a/README_zh.md +++ b/README_zh.md @@ -40,9 +40,16 @@ cd HuixiangDou && mkdir workdir # 创建工作目录 python3 -m pip install -r requirements.txt # 安装依赖，python3.11 需要 `conda install conda-forge::faiss-gpu` python3 service/feature_store.py # 把 repodir 的特征保存到 workdir ``` -首次运行将自动下载配置中的 [text2vec-large-chinese](https://huggingface.co/GanymedeNil/text2vec-large-chinese)，如果自动下载失败，可以手动下载到本地，然后在 `config.ini` 设置模型路径。 +首次运行将自动下载配置中的 [text2vec-large-chinese](https://huggingface.co/GanymedeNil/text2vec-large-chinese)。考虑到不同地区 huggingface 连接问题，建议先手动下载到本地，然后在 `config.ini` 设置模型路径。例如： -结束后，茴香豆能够区分应该处理哪些用户话题，哪些闲聊应该拒绝。请编辑 [good_questions](./resource/good_questions.json) 和 [bad_questions](./resource/bad_questions.json)，尝试自己的领域知识（医疗，金融，电力等）。 +```shell +# config.ini +[feature_store] +.. +model_path = "/path/to/text2vec-large-chinese" +``` + +运行结束后，茴香豆能够区分应该处理哪些用户话题，哪些闲聊应该拒绝。请编辑 [good_questions](./resource/good_questions.json) 和 [bad_questions](./resource/bad_questions.json)，尝试自己的领域知识（医疗，金融，电力等）。 ```shell # 接受技术话题 diff --git a/requirements.txt b/requirements.txt index 36ce7a51..d62317ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,10 @@ sentence_transformers readability-lxml faiss-gpu transformers>=4.33.2 +openai>=1.0.0 +pydantic>=1.10.13 +transformers_stream_generator +accelerate>=0.26.1 +beautifulsoup4 +tiktoken +einops diff --git a/service/llm_server_hybrid.py b/service/llm_server_hybrid.py index 4a0741af..1d588237 100644 --- a/service/llm_server_hybrid.py +++ b/service/llm_server_hybrid.py @@ -43,7 +43,8 @@ def __init__(self, model_path, trust_remote_code=True, device_map="auto", - # fp16=True, +# torch_dtype="auto", +# fp16=True, ).eval() else: logger.warning(f'local LLM disabled.') diff --git a/service/worker.py b/service/worker.py index d80e8fcd..32895a95 100644 --- a/service/worker.py +++ b/service/worker.py @@ -136,7 +136,7 @@ def generate(self, query, history, groupname): response = self.llm.generate_response(prompt=prompt, history=history, remote=True) - tracker.track('feature store doc', [db_context_part, response]) + tracker.log('feature store doc', [db_context_part, response]) return ErrorCode.SUCCESS, response else: diff --git a/tests/test_hf_import_accelerate.py b/tests/test_hf_import_accelerate.py new file mode 100644 index 00000000..c37181ac --- /dev/null +++ b/tests/test_hf_import_accelerate.py @@ -0,0 +1,12 @@ +from accelerate import dispatch_model, infer_auto_device_map, init_empty_weights +from accelerate.hooks import add_hook_to_module +from accelerate.utils import ( + check_tied_parameters_on_same_device, + find_tied_parameters, + get_balanced_memory, + get_max_memory, + load_offloaded_weights, + offload_weight, + save_offload_index, + set_module_tensor_to_device, +) From 7a0e2f3da5433ce5400ab4e31b8c65ce7cf64a3a Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Fri, 12 Jan 2024 10:43:50 +0800 Subject: [PATCH 2/4] docs(project): update --- README.md | 8 ++++---- README_zh.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0edef52c..ab9efa6d 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ View [HuixiangDou inside](./huixiangdou-inside.md). The following are the hardware requirements for running. It is suggested to follow this document, starting with the basic version and gradually experiencing advanced features. -| Version | GPU Memory Requirements | Remarks | Tested on | +| Version | GPU Memory Requirements | Features | Tested on | | :-: | :-: | :-: | :-: | -| Basic Version | 20GB | Able to answer basic domain knowledge questions, zero cost operation | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | -| Advanced Version | 40GB | Able to answer source code level questions, zero cost operation | ![](https://img.shields.io/badge/linux%20A100%2080G-passed-blue?style=for-the-badge) | -| Modified Version | 4GB | Using openai API to replace local LLM, basic development capability required, operation involves cost | ![](https://img.shields.io/badge/linux%201660ti%206GB-passed-blue?style=for-the-badge) | +| Basic Version | 20GB | Answer basic domain knowledge questions, zero cost | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | +| Advanced Version | 40GB | Answer source code level questions, zero cost | ![](https://img.shields.io/badge/linux%20A100%2080G-passed-blue?style=for-the-badge) | +| Modified Version | 4GB | Using openai API, operation involves cost | ![](https://img.shields.io/badge/linux%201660ti%206GB-passed-blue?style=for-the-badge) | # 🔥 Run diff --git a/README_zh.md b/README_zh.md index 22a47298..3d9a472b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -18,9 +18,9 @@ 以下是运行茴香豆的硬件需求。建议遵循部署流程，从基础版开始，逐渐体验高级特性。 -| 版本 | 硬件需求 | 备注 | 已验证设备 | +| 版本 | 硬件需求 | 描述 | 已验证设备 | | :-: | :-: | :-: | :-: | -| 基础版 | 20GB | 能够回答领域知识的基础问题，零成本运行 | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | +| 基础版 | 20GB | 能回答领域知识的基础问题，零成本运行 | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | | 高级版 | 40GB | 能够回答源码级问题，零成本运行 | ![](https://img.shields.io/badge/linux%20A100%2080G-passed-blue?style=for-the-badge) | | 魔改版 | 4GB| 用 openai API 替代本地 LLM，需要基础开发能力，运行需要费用 | ![](https://img.shields.io/badge/linux%201660ti%206GB-passed-blue?style=for-the-badge) | From 7814a72ce91efa5bb57367ee19df298bad4c5cf2 Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Fri, 12 Jan 2024 10:46:04 +0800 Subject: [PATCH 3/4] docs(project): update --- README.md | 8 ++++---- README_zh.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ab9efa6d..38418fea 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ View [HuixiangDou inside](./huixiangdou-inside.md). The following are the hardware requirements for running. It is suggested to follow this document, starting with the basic version and gradually experiencing advanced features. -| Version | GPU Memory Requirements | Features | Tested on | +| Version | GPU Memory Requirements | Features | Tested on Linux | | :-: | :-: | :-: | :-: | -| Basic Version | 20GB | Answer basic domain knowledge questions, zero cost | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | -| Advanced Version | 40GB | Answer source code level questions, zero cost | ![](https://img.shields.io/badge/linux%20A100%2080G-passed-blue?style=for-the-badge) | -| Modified Version | 4GB | Using openai API, operation involves cost | ![](https://img.shields.io/badge/linux%201660ti%206GB-passed-blue?style=for-the-badge) | +| Basic Version | 20GB | Answer basic domain knowledge questions, zero cost | ![](https://img.shields.io/badge/3090%2024G-passed-blue?style=for-the-badge) | +| Advanced Version | 40GB | Answer source code level questions, zero cost | ![](https://img.shields.io/badge/A100%2080G-passed-blue?style=for-the-badge) | +| Modified Version | 4GB | Using openai API, operation involves cost | ![](https://img.shields.io/badge/1660ti%206GB-passed-blue?style=for-the-badge) | # 🔥 Run diff --git a/README_zh.md b/README_zh.md index 3d9a472b..3132c26f 100644 --- a/README_zh.md +++ b/README_zh.md @@ -18,11 +18,11 @@ 以下是运行茴香豆的硬件需求。建议遵循部署流程，从基础版开始，逐渐体验高级特性。 -| 版本 | 硬件需求 | 描述 | 已验证设备 | +| 版本 | 硬件需求 | 描述 | Linux 系统已验证设备 | | :-: | :-: | :-: | :-: | -| 基础版 | 20GB | 能回答领域知识的基础问题，零成本运行 | ![](https://img.shields.io/badge/linux%203090%2024G-passed-blue?style=for-the-badge) | -| 高级版 | 40GB | 能够回答源码级问题，零成本运行 | ![](https://img.shields.io/badge/linux%20A100%2080G-passed-blue?style=for-the-badge) | -| 魔改版 | 4GB| 用 openai API 替代本地 LLM，需要基础开发能力，运行需要费用 | ![](https://img.shields.io/badge/linux%201660ti%206GB-passed-blue?style=for-the-badge) | +| 基础版 | 20GB | 能回答领域知识的基础问题，零成本运行 | ![](https://img.shields.io/badge/3090%2024G-passed-blue?style=for-the-badge) | +| 高级版 | 40GB | 能够回答源码级问题，零成本运行 | ![](https://img.shields.io/badge/A100%2080G-passed-blue?style=for-the-badge) | +| 魔改版 | 4GB| 用 openai API 替代本地 LLM，需要基础开发能力，运行需要费用 | ![](https://img.shields.io/badge/1660ti%206GB-passed-blue?style=for-the-badge) | # 🔥 运行 From ffbc4019cb3c3fc6e3752b09a18975a61cb47636 Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Fri, 12 Jan 2024 10:48:03 +0800 Subject: [PATCH 4/4] docs(project): update --- README.md | 2 +- README_zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38418fea..a337e696 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The following are the hardware requirements for running. It is suggested to foll | :-: | :-: | :-: | :-: | | Basic Version | 20GB | Answer basic domain knowledge questions, zero cost | ![](https://img.shields.io/badge/3090%2024G-passed-blue?style=for-the-badge) | | Advanced Version | 40GB | Answer source code level questions, zero cost | ![](https://img.shields.io/badge/A100%2080G-passed-blue?style=for-the-badge) | -| Modified Version | 4GB | Using openai API, operation involves cost | ![](https://img.shields.io/badge/1660ti%206GB-passed-blue?style=for-the-badge) | +| Modified Version | 4GB | Using openai API, operation involves cost | ![](https://img.shields.io/badge/1660ti%206G-passed-blue?style=for-the-badge) | # 🔥 Run diff --git a/README_zh.md b/README_zh.md index 3132c26f..f9616776 100644 --- a/README_zh.md +++ b/README_zh.md @@ -22,7 +22,7 @@ | :-: | :-: | :-: | :-: | | 基础版 | 20GB | 能回答领域知识的基础问题，零成本运行 | ![](https://img.shields.io/badge/3090%2024G-passed-blue?style=for-the-badge) | | 高级版 | 40GB | 能够回答源码级问题，零成本运行 | ![](https://img.shields.io/badge/A100%2080G-passed-blue?style=for-the-badge) | -| 魔改版 | 4GB| 用 openai API 替代本地 LLM，需要基础开发能力，运行需要费用 | ![](https://img.shields.io/badge/1660ti%206GB-passed-blue?style=for-the-badge) | +| 魔改版 | 4GB| 用 openai API 替代本地 LLM，需要基础开发能力，运行需要费用 | ![](https://img.shields.io/badge/1660ti%206G-passed-blue?style=for-the-badge) | # 🔥 运行