From 68c3fbb83118960c03ed70bb57777ab5ec7963ea Mon Sep 17 00:00:00 2001 From: Pouya Rostam Date: Wed, 8 May 2024 11:58:56 -0700 Subject: [PATCH] rev 1 --- .github/workflows/c-demos.yml | 6 +- demo/c/README.md | 123 ++++++++++--------------------- demo/c/picollm_demo_completion.c | 31 ++++---- 3 files changed, 55 insertions(+), 105 deletions(-) diff --git a/.github/workflows/c-demos.yml b/.github/workflows/c-demos.yml index ccaf88c..e21704a 100644 --- a/.github/workflows/c-demos.yml +++ b/.github/workflows/c-demos.yml @@ -9,8 +9,6 @@ on: - '.github/workflows/c-demos.yml' - 'demo/c/**' - 'include/**' - - 'lib/common/**' - - 'lib/jetson/**' - 'lib/linux/**' - 'lib/mac/**' - 'lib/raspberry-pi/**' @@ -23,8 +21,6 @@ on: - '.github/workflows/c-demos.yml' - 'demo/c/**' - 'include/**' - - 'lib/common/**' - - 'lib/jetson/**' - 'lib/linux/**' - 'lib/mac/**' - 'lib/raspberry-pi/**' @@ -44,7 +40,7 @@ jobs: matrix: xpu: [ cpu ] make_file: [ "Unix Makefiles" ] - machine: [ rpi4-32, rpi4-64, rpi5-32, rpi5-64, pv-linux, pv-ios ] + machine: [ rpi3-32, rpi3-64, rpi4-32, rpi4-64, rpi5-32, rpi5-64, pv-linux, pv-ios ] include: - xpu: gpu machine: pv-linux diff --git a/demo/c/README.md b/demo/c/README.md index 1470041..2aac3d2 100644 --- a/demo/c/README.md +++ b/demo/c/README.md @@ -24,31 +24,31 @@ picoLLM Inference Engine supports the following open-weight models. The models a [Picovoice Console](https://console.picovoice.ai/). - Gemma - - `gemma-2b` - - `gemma-2b-it` - - `gemma-7b` - - `gemma-7b-it` + - `gemma-2b` + - `gemma-2b-it` + - `gemma-7b` + - `gemma-7b-it` - Llama-2 - - `llama-2-7b` - - `llama-2-7b-chat` - - `llama-2-13b` - - `llama-2-13b-chat` - - `llama-2-70b` - - `llama-2-70b-chat` + - `llama-2-7b` + - `llama-2-7b-chat` + - `llama-2-13b` + - `llama-2-13b-chat` + - `llama-2-70b` + - `llama-2-70b-chat` - Llama-3 - - `llama-3-8b` - - `llama-3-8b-instruct` - - `llama-3-70b` - - `llama-3-70b-instruct` + - `llama-3-8b` + - `llama-3-8b-instruct` + - `llama-3-70b` + - `llama-3-70b-instruct` - Mistral - - `mistral-7b-v0.1` - - `mistral-7b-instruct-v0.1` - - `mistral-7b-instruct-v0.2` + - `mistral-7b-v0.1` + - `mistral-7b-instruct-v0.1` + - `mistral-7b-instruct-v0.2` - Mixtral - - `mixtral-8x7b-v0.1` - - `mixtral-8x7b-instruct-v0.1` + - `mixtral-8x7b-v0.1` + - `mixtral-8x7b-instruct-v0.1` - Phi-2 - - `phi2` + - `phi2` ## Usage @@ -77,46 +77,39 @@ cmake --build demo/c/build Running the demo without arguments prints the usage: ```console -Usage: ./demo/c/build/picollm_demo_completion -a ACCESS_KEY -l LIBRARY_PATH -m MODEL_PATH -[-d DEVICE] [-c COMPLETION_TOKEN_LIMIT] [-s STOP_PHRASES] [-e SEED] [-r PRESENCE_PENALTY] -[-f FREQUENCY_PENALTY] [-o TOP_P] [-t TEMPERATURE] [-n MAX_OUTPUT_TOKENS] [-c NUM_TOP_CHOICES] +Usage: ./demo/c/build/picollm_demo_completion -a ACCESS_KEY -l LIBRARY_PATH -m MODEL_PATH +[-d DEVICE] [-c COMPLETION_TOKEN_LIMIT] [-s STOP_PHRASES] [-e SEED] [-r PRESENCE_PENALTY] +[-f FREQUENCY_PENALTY] [-o TOP_P] [-t TEMPERATURE] [-n MAX_OUTPUT_TOKENS] [-c NUM_TOP_CHOICES] [-v] [-h] -p PROMPT -v: enable verbose output -h: show available devices ``` -Run the command corresponding to your platform from the root of the repository. Replace `${ACCESS_KEY}` with yours obtained from Picovoice Console, `${MODEL_PATH}` with the path to a model file -downloaded from Picovoice Console, and `${PROMPT}` with a prompt string. `-v` enables verbose output, and `-h` shows available devices. For more information on the optional parameters, see the [picollm header file](../../include/pv_picollm.h). +For a simle completion demo, run the command corresponding to your platform from the root of the repository. Replace `${ACCESS_KEY}` with yours obtained from Picovoice Console, `${MODEL_PATH}` with the path to a model file downloaded from Picovoice Console, and `${PROMPT}` with a prompt string. `-v` enables verbose output, and `-h` shows available devices. For more information on the optional parameters, see the [picollm header file](../../include/pv_picollm.h). -#### Linux (x86_64) +#### Linux/macOS ```console ./demo/c/build/picollm_demo_completion \ -a ${ACCESS_KEY} \ -m ${MODEL_PATH} \ --l lib/linux/x86_64/libpv_picollm.so \ +-l ${LIBRARY_PATH} \ -p ${PROMPT} ``` -#### macOS (x86_64) +where `${LIBRARY_PATH}` is the path to the Picovoice library file corresponding to your platform shown in the table below: -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/mac/x86_64/libpv_picollm.dylib \ --p ${PROMPT} -``` - -#### macOS (arm64) - -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/mac/arm64/libpv_picollm.dylib \ --p ${PROMPT} -``` +| Platform | Library Path | +| ----------------------- | ---------------------------------------------------- | +| Linux (x86_64) | lib/linux/x86_64/libpv_picollm.so | +| macOS (x86_64) | lib/mac/x86_64/libpv_picollm.dylib | +| macOS (arm64) | lib/mac/arm64/libpv_picollm.dylib | +| Raspberry Pi 5 | lib/raspberry-pi/cortex-a76/libpv_picollm.so | +| Raspberry Pi 5 (64-bit) | lib/raspberry-pi/cortex-a76-aarch64/libpv_picollm.so | +| Raspberry Pi 4 | lib/raspberry-pi/cortex-a76/libpv_picollm.so | +| Raspberry Pi 4 (64-bit) | lib/raspberry-pi/cortex-a76-aarch64/libpv_picollm.so | +| Raspberry Pi 3 | lib/raspberry-pi/cortex-a53/libpv_picollm.so | +| Raspberry Pi 3 (64-bit) | lib/raspberry-pi/cortex-a53-aarch64/libpv_picollm.so | #### Windows @@ -129,43 +122,3 @@ demo\c\build\picollm_demo_completion.exe ^ -l lib\windows\amd64\libpv_picollm.dll ^ -p ${PROMPT} ``` - -#### Raspberry Pi 4 - -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/raspberry-pi/cortex-a72/libpv_picollm.so \ --p ${PROMPT} -``` - -#### Raspberry Pi 4 (64-bit) - -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/raspberry-pi/cortex-a72-aarch64/libpv_picollm.so \ --p ${PROMPT} -``` - -#### Raspberry Pi 3 - -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/raspberry-pi/cortex-a53/libpv_picollm.so \ --p ${PROMPT} -``` - -#### Raspberry Pi 3 (64-bit) - -```console -./demo/c/build/picollm_demo_completion \ --a ${ACCESS_KEY} \ --m ${MODEL_PATH} \ --l lib/raspberry-pi/cortex-a53-aarch64/libpv_picollm.so \ --p ${PROMPT} -``` \ No newline at end of file diff --git a/demo/c/picollm_demo_completion.c b/demo/c/picollm_demo_completion.c index 5503b62..a30685a 100644 --- a/demo/c/picollm_demo_completion.c +++ b/demo/c/picollm_demo_completion.c @@ -1,5 +1,5 @@ /* - Copyright 2023-2024 Picovoice Inc. + Copyright 2024 Picovoice Inc. You may not use this file except in compliance with the license. A copy of the license is located in the "LICENSE" file accompanying this source. @@ -302,56 +302,57 @@ int picovoice_main(int argc, char **argv) { exit(EXIT_FAILURE); } - pv_status_t (*pv_picollm_delete_completion_tokens_func)(pv_picollm_completion_token_t *, int32_t) = - load_symbol(dl_handle, "pv_picollm_delete_completion_tokens"); + pv_status_t (*pv_picollm_delete_completion_tokens_func)(pv_picollm_completion_token_t *, int32_t) = + load_symbol(dl_handle, "pv_picollm_delete_completion_tokens"); if (!pv_picollm_delete_completion_tokens_func) { print_dl_error("failed to load `pv_picollm_delete_completion_tokens`"); exit(EXIT_FAILURE); } - pv_status_t (*pv_picollm_delete_completion_func)(char *) = - load_symbol(dl_handle, "pv_picollm_delete_completion"); + pv_status_t (*pv_picollm_delete_completion_func)(char *) = + load_symbol(dl_handle, "pv_picollm_delete_completion"); if (!pv_picollm_delete_completion_func) { print_dl_error("failed to load `pv_picollm_delete_completion`"); exit(EXIT_FAILURE); } - const char *(*pv_picollm_version_func)(void) = load_symbol(dl_handle, "pv_picollm_version"); + const char *(*pv_picollm_version_func)(void) = + load_symbol(dl_handle, "pv_picollm_version"); if (!pv_picollm_version_func) { print_dl_error("failed to load `pv_picollm_version`"); exit(EXIT_FAILURE); } - pv_status_t (*pv_picollm_model_func)(pv_picollm_t *, char **) = load_symbol(dl_handle, "pv_picollm_model"); + pv_status_t (*pv_picollm_model_func)(pv_picollm_t *, char **) = + load_symbol(dl_handle, "pv_picollm_model"); if (!pv_picollm_model_func) { print_dl_error("failed to load `pv_picollm_model`"); exit(EXIT_FAILURE); } - pv_status_t (*pv_picollm_list_hardware_devices_func)(char ***, int32_t *) = load_symbol(dl_handle, - "pv_picollm_list_hardware_devices"); + pv_status_t (*pv_picollm_list_hardware_devices_func)(char ***, int32_t *) = + load_symbol(dl_handle, "pv_picollm_list_hardware_devices"); if (!pv_picollm_list_hardware_devices_func) { print_dl_error("failed to load `pv_picollm_list_hardware_devices`"); exit(EXIT_FAILURE); } - pv_status_t (*pv_picollm_free_hardware_devices_func)(char **, int32_t) = load_symbol(dl_handle, - "pv_picollm_free_hardware_devices"); + pv_status_t (*pv_picollm_free_hardware_devices_func)(char **, int32_t) = + load_symbol(dl_handle, "pv_picollm_free_hardware_devices"); if (!pv_picollm_free_hardware_devices_func) { print_dl_error("failed to load `pv_picollm_free_hardware_devices`"); exit(EXIT_FAILURE); } - pv_status_t (*pv_get_error_stack_func)( - char ***, - int32_t *) = load_symbol(dl_handle, "pv_get_error_stack"); + pv_status_t (*pv_get_error_stack_func)(char ***, int32_t *) = + load_symbol(dl_handle, "pv_get_error_stack"); if (!pv_get_error_stack_func) { print_dl_error("failed to load 'pv_get_error_stack_func'"); exit(EXIT_FAILURE); } void (*pv_free_error_stack_func)(char **) = - load_symbol(dl_handle, "pv_free_error_stack"); + load_symbol(dl_handle, "pv_free_error_stack"); if (!pv_free_error_stack_func) { print_dl_error("failed to load 'pv_free_error_stack_func'"); exit(EXIT_FAILURE);