From a3abf684355e4dc367444197e11cc01d19286f59 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Fri, 3 Jul 2026 15:55:47 +0200 Subject: [PATCH 01/11] Kokoro usage improvements --- Dockerfile.redhat | 6 + Dockerfile.ubuntu | 6 + Makefile | 12 +- common_settings.bzl | 3 +- create_package.sh | 1 + demos/audio/README.md | 157 +++++---------------- demos/benchmark/v3/benchmark.py | 74 +++++++--- demos/common/export_models/export_model.py | 15 +- distro.bzl | 26 +--- docs/build_from_source.md | 26 ++++ prepare_llm_models.sh | 14 +- src/BUILD | 6 - src/audio/text_to_speech/t2s_servable.cpp | 54 +++++++ src/graph_export/graph_export.cpp | 30 ---- src/test/audio/graph_tts.pbtxt | 8 +- src/test/audio/text2speech_test.cpp | 49 +++++-- src/test/graph_export_test.cpp | 6 - third_party/BUILD | 19 +-- windows_build.bat | 10 +- windows_prepare_llm_models.bat | 10 +- 20 files changed, 247 insertions(+), 285 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 04fe81cbea..561b732604 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -315,6 +315,12 @@ RUN rm -f /usr/lib64/cmake/OpenSSL/OpenSSLConfig.cmake # hadolint ignore=SC2046 RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi) +# espeak-ng is built as a separate step, independent of the OVMS binary. +# Set ESPEAK=0 to skip the espeak build. +ARG ESPEAK=1 +# hadolint ignore=DL3059 +RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi + # Tests execution COPY ci/check_coverage.bat /ovms/ ARG CHECK_COVERAGE=0 diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index ed38a3cb02..59a7fbdf23 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -331,6 +331,12 @@ ARG OPTIMIZE_BUILDING_TESTS=0 # hadolint ignore=SC2046 RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi; +# espeak-ng is built as a separate step, independent of the OVMS binary. +# Set ESPEAK=0 to skip the espeak build. +ARG ESPEAK=1 +# hadolint ignore=DL3059 +RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi + ARG RUN_TESTS=0 RUN if [ "$RUN_TESTS" == "1" ] ; then mkdir -p demos/common/export_models/ && mv export_model.py demos/common/export_models/ && ./prepare_llm_models.sh /ovms/src/test/llm_testing docker && ./run_unit_tests.sh ; fi diff --git a/Makefile b/Makefile index 654875576c..ff800383ca 100644 --- a/Makefile +++ b/Makefile @@ -148,13 +148,8 @@ else ifeq ($(findstring redhat,$(BASE_OS)),redhat) else $(error BASE_OS must be either ubuntu or redhat) endif -ifeq ($(ESPEAK),1) - ESPEAK_PARAMS = " --//:espeak=on" -else - ESPEAK_PARAMS = " --//:espeak=off" -endif -CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --config=mp_off_py_off"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(ESPEAK_PARAMS) -BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(ESPEAK_PARAMS)$(REPO_ENV) +CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --config=mp_off_py_off"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) +BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS)$(REPO_ENV) # Option to Override release image. # Release image OS *must have* glibc version >= glibc version on BASE_OS: @@ -249,7 +244,8 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg JOBS=$(JOBS)\ --build-arg CAPI_FLAGS=$(CAPI_FLAGS)\ --build-arg VERBOSE_LOGS=$(VERBOSE_LOGS)\ - --build-arg KONFLUX=$(KONFLUX) + --build-arg KONFLUX=$(KONFLUX)\ + --build-arg ESPEAK=$(ESPEAK) .PHONY: default docker_build \ diff --git a/common_settings.bzl b/common_settings.bzl index 8818ec83fe..b2f1101f2f 100644 --- a/common_settings.bzl +++ b/common_settings.bzl @@ -20,7 +20,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@mediapipe//mediapipe/framework:more_selects.bzl", "more_selects") load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("//:distro.bzl", "distro_flag", "espeak_flag") +load("//:distro.bzl", "distro_flag") # cc_library rule wrapper that will accept the same arguments but if user will not provide # copts, linkopts, local_defines it will set them to the defaults @@ -58,7 +58,6 @@ def ovms_cc_library(**kwargs): def create_config_settings(): distro_flag() - espeak_flag() native.config_setting( name = "disable_mediapipe", define_values = { diff --git a/create_package.sh b/create_package.sh index 69812d7302..6220a93ec9 100755 --- a/create_package.sh +++ b/create_package.sh @@ -36,6 +36,7 @@ ESPEAK_DATA_SRC=$(find /ovms/bazel-out/k8-*/bin/external/espeak_ng -type d -name if [ -n "$ESPEAK_DATA_SRC" ] && [ -d "$ESPEAK_DATA_SRC" ] ; then mkdir -p /ovms_release/share cp -rL "$ESPEAK_DATA_SRC" /ovms_release/share/ ; + chmod -R u+w /ovms_release/share/espeak-ng-data 2>/dev/null || true ; fi # Resolve the packaged eSpeak shared object dynamically so version bumps # do not require touching this script. diff --git a/demos/audio/README.md b/demos/audio/README.md index a59cd77c11..07a4333668 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -8,116 +8,46 @@ Check supported [Speech Recognition Models](https://openvinotoolkit.github.io/op ## Prerequisites -**OVMS version 2025.4** This demo requires version 2025.4 or nightly release. - -**Model preparation**: Python 3.10 or higher with pip +**OVMS version 2026.3** This demo requires version 2026.3 or nightly release. **Model Server deployment**: Installed Docker Engine or OVMS binary package according to the [baremetal deployment guide](../../docs/deploying_server_baremetal.md) **Client**: curl or Python for using OpenAI client package ## Speech generation -### Prepare speaker embeddings -When generating speech you can use default speaker voice or you can prepare your own speaker embedding file. Here you can see how to do it with downloaded file from online repository, but you can try with your own speech recording as well: -```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/requirements.txt -mkdir audio_samples -curl --create-dirs "https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0032_8k.wav" -o audio_samples/audio.wav -curl --create-dirs https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/create_speaker_embedding.py -o models/speakers/create_speaker_embedding.py -python models/speakers/create_speaker_embedding.py audio_samples/audio.wav models/speakers/voice1.bin -``` - +Kokoro is the primary example in this demo, but SpeechT5 remains supported for existing deployments. ### Model preparation -Supported models should use the topology of [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts) which needs to be converted to IR format before using in OVMS. - -Specific OVMS pull mode example for models requiring conversion is described in the [Ovms pull mode](../../docs/pull_hf_models.md#pulling-models-outside-openvino-organization) - -Or you can use the python export_model.py script described below. - -Here, the original Text to Speech model will be converted to IR format and optionally quantized. -That ensures faster initialization time, better performance and lower memory consumption. -Execution parameters will be defined inside the `graph.pbtxt` file. - -Download export script, install it's dependencies and create directory for the models: -```console -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt -mkdir models -``` - -Run `export_model.py` script to download and quantize the model: - -> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before running the export script to connect to the HF Hub. -> **Note:** Exporting `microsoft/speecht5_tts` model requires Python 3.10 - -**CPU** -```console -python export_model.py text2speech --source_model microsoft/speecht5_tts --weight-format fp16 --model_name microsoft/speecht5_tts --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan --speaker_name voice1 --speaker_path models/speakers/voice1.bin -``` +This demo uses a pre-exported OpenVINO IR model [luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS](https://huggingface.co/luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS) available on HuggingFace. +The model can be pulled directly by OVMS without any conversion step. -> **Note:** Change the `--weight-format` to quantize the model to `int8` precision to reduce memory consumption and improve performance. -> **Note:** `speaker_name` and `speaker_path` may be omitted if the default model voice is sufficient +> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before starting OVMS to connect to the HF Hub. +> **Note:** Kokoro voices are loaded from the `voices/` directory in the model repository. OVMS loads each voice file under the filename without the extension, for example `af_alloy` for `af_alloy.bin`. -The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. +See the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. ### Deployment -**CPU** +**Deploying with Docker** -Running this command starts the container with CPU only target device: ```bash -mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts +mkdir models +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path /models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU --task text2speech ``` **Deploying on Bare Metal** ```bat -mkdir models -ovms --rest_port 8000 --model_path models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts +mkdir c:\models +ovms --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path c:\models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU --task text2speech ``` ### Request Generation -:::{dropdown} **Unary call with curl with default voice** - - -```bash -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav -``` -::: - -:::{dropdown} **Unary call with OpenAI python library with default voice** - -```python -from pathlib import Path -from openai import OpenAI - -prompt = "The quick brown fox jumped over the lazy dog" -filename = "speech.wav" -url="http://localhost:8000/v3" - - -speech_file_path = Path(__file__).parent / "speech.wav" -client = OpenAI(base_url=url, api_key="not_used") - -with client.audio.speech.with_streaming_response.create( - model="microsoft/speecht5_tts", - voice=None, - input=prompt -) as response: - response.stream_to_file(speech_file_path) - - -print("Generation finished") -``` -::: - :::{dropdown} **Unary call with curl** ```bash -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"voice\":\"voice1\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"af_alloy\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav ``` ::: @@ -136,8 +66,8 @@ speech_file_path = Path(__file__).parent / "speech.wav" client = OpenAI(base_url=url, api_key="not_used") with client.audio.speech.with_streaming_response.create( - model="microsoft/speecht5_tts", - voice="voice1", + model="Kokoro-82M-OpenVINO-FP16-OVMS", + voice="af_alloy", input=prompt ) as response: response.stream_to_file(speech_file_path) @@ -152,20 +82,25 @@ Play speech.wav file to check generated speech. ## Benchmarking speech generation An asynchronous benchmarking client can be used to access the model server performance with various load conditions. Below are execution examples captured on Intel(R) Core(TM) Ultra 7 258V. +> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. +> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. + ```console pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py -python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model microsoft/speecht5_tts --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --tokenizer OpenVINO/whisper-large-v3-turbo-fp16-ov --trust-remote-code True -Number of documents: 100 -100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [01:58<00:00, 1.19s/it] -Tokens: 1802 -Success rate: 100.0%. (100/100) -Throughput - Tokens per second: 15.2 -Mean latency: 63653.98 ms -Median latency: 66736.83 ms -Average document length: 18.02 tokens +python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model Kokoro-82M-OpenVINO-FP16-OVMS --batch_size 1 --limit 1000 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --voice af_alloy +Number of documents: 1000 +100%|█████████████████████████████████████████████████████████████████████████████████| 1000/1000 [16:37<00:00, 1.00it/s] +Success rate: 100.0%. (1000/1000) +Mean latency: 1506.60 ms +Median latency: 1072.72 ms +Mean RTFx: 3.655x +Median RTFx: 3.899x ``` +> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. +> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. + ## Transcription ### Model preparation Many variances of Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. @@ -202,12 +137,10 @@ ovms --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v ``` ::: -> **Note:** Sentence timestamps are supported via `timestamp_granularities[]=segment` in transcription requests. - The default configuration should work in most cases but the parameters can be tuned via OVMS arguments. See the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. ### Request Generation -Transcript file that was previously generated with audio/speech endpoint. +Transcribe the speech.wav file generated in the [Speech generation](#speech-generation) section. > **Note:** Streaming responses are supported for `audio/transcriptions`. `audio/translations` does not support streaming. @@ -437,33 +370,13 @@ Where: You can replace `librispeech` with other datasets supported by the leaderboard configuration. For multilingual models run_eval_ml.py should be used. ## Translation -To test translations endpoint we first need to prepare audio file with speech in language other than English, e.g. Spanish. To generate such sample we will use finetuned version of microsoft/speecht5_tts model. +To test the translations endpoint we first need to prepare an audio file with speech in a language other than English, e.g. Spanish. To generate such a sample, follow the [Speech generation](#speech-generation) section to deploy Kokoro and then run: -**Deploying with Docker** - -```bash -mkdir -p models - -python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan - -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/speecht5_tts_spanish --model_name speecht5_tts_spanish - -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav -``` - -**Deploying on Bare Metal** - -```bat -mkdir models - -python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan - -ovms --rest_port 8000 --model_path models/speecht5_tts_spanish --model_name speecht5_tts_spanish - -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav +```console +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"em_alex\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav ``` -### Model preparation +### Deployment Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. Here is an example of OpenVINO/whisper-large-v3-fp16-ov deployment: @@ -504,7 +417,7 @@ ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_r ::: ### Request Generation -Transcript and translate file that was previously generated with audio/speech endpoint. +Translate the speech_spanish.wav file generated above. :::{dropdown} **Unary call with cURL** diff --git a/demos/benchmark/v3/benchmark.py b/demos/benchmark/v3/benchmark.py index 88c9aa8f18..b6fdeb3ad6 100644 --- a/demos/benchmark/v3/benchmark.py +++ b/demos/benchmark/v3/benchmark.py @@ -52,6 +52,7 @@ parser.add_argument('--model', required=False, default='Alibaba-NLP/gte-large-en-v1.5', help='HF model name', dest='model') parser.add_argument('--request_rate', required=False, default='inf', help='Average amount of requests per seconds in random distribution', dest='request_rate') parser.add_argument('--batch_size', required=False, type=int, default=16, help='Number of strings in every requests', dest='batch_size') +parser.add_argument('--voice', required=False, help='Voice name for text2speech backend', dest='voice') parser.add_argument('--backend', required=False, default='ovms-embeddings', choices=['ovms-embeddings','tei-embed','infinity-embeddings','ovms_rerank','text2speech','speech2text', 'translations'], help='Backend serving API type', dest='backend') parser.add_argument('--limit', required=False, type=int, default=1000, help='Number of documents to use in testing', dest='limit') parser.add_argument('--split', required=False, default='train', help='Dataset split', dest='split') @@ -104,6 +105,7 @@ class RequestFuncOutput: success: bool = False latency: float = 0.0 tokens_len: int = 0 + audio_duration: float = 0.0 error: str = "" text: str = "" @@ -128,10 +130,13 @@ async def async_request_text2speech( "model": request_func_input.model, "input": request_func_input.documents[0], } + if args["voice"] is not None: + payload["voice"] = args["voice"] headers = application_json_headers output = RequestFuncOutput() st = time.perf_counter() + audio_bytes = bytearray() try: async with session.post(url=api_url, json=payload, headers=headers) as response: @@ -139,13 +144,14 @@ async def async_request_text2speech( async for chunk_bytes in response.content: if not chunk_bytes: continue - # uncomment for response debugging - # chunk_bytes = chunk_bytes.decode("utf-8") - # data = json.loads(chunk_bytes) - # TBD: saving response to file - timestamp = time.perf_counter() - output.success = True - output.latency = timestamp - st + audio_bytes.extend(chunk_bytes) + output.success = True + output.latency = time.perf_counter() - st + try: + output.audio_duration = soundfile.info(io.BytesIO(audio_bytes)).duration + except Exception as e: + output.success = False + output.error = f"Could not decode WAV response to compute audio duration: {e}" else: output.error = response.reason or "" output.success = False @@ -387,6 +393,7 @@ async def limited_request_func(request_func_input, pbar): "latencies": [output.latency for output in outputs], "successes": [output.success for output in outputs], "token_count": [output.tokens_len for output in outputs], + "audio_durations": [output.audio_duration for output in outputs], } return result @@ -428,18 +435,47 @@ async def limited_request_func(request_func_input, pbar): args["api_url"] = default_api_url benchmark_results = asyncio.run(benchmark(docs=docs, model=args["model"], api_url=args["api_url"], request_rate=float(args["request_rate"]), backend_function=backend_function)) -if args["backend"] == "speech2text" or args["backend"] == "translations": + +success_count = sum(benchmark_results['successes']) +total_count = len(benchmark_results['successes']) +success_rate = (success_count / total_count * 100.0) if total_count else 0.0 + +if args["backend"] == "text2speech": + rtfx_values = [] + for latency, audio_duration, success in zip( + benchmark_results['latencies'], + benchmark_results['audio_durations'], + benchmark_results['successes'], + ): + if success and latency > 0 and audio_duration > 0: + # RTFx = generated audio seconds per one second of processing time. + rtfx_values.append(audio_duration / latency) + + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + + if len(rtfx_values) > 0: + print(f"Mean RTFx: {np.mean(rtfx_values):.3f}x") + print(f"Median RTFx: {np.median(rtfx_values):.3f}x") + else: + print("Mean RTFx: n/a") + print("Median RTFx: n/a") +elif args["backend"] == "speech2text" or args["backend"] == "translations": num_tokens = sum(benchmark_results['token_count']) + #print(benchmark_results) + print("Tokens:",num_tokens) + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Average document length: {num_tokens / len(docs)} tokens") else: num_tokens = count_tokens(docs=docs,model=args["model"]) -#print(benchmark_results) -print("Tokens:",num_tokens) -print(f"Success rate: {sum(benchmark_results['successes'])/len(benchmark_results['successes'])*100}%. ({sum(benchmark_results['successes'])}/{len(benchmark_results['successes'])})") -print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") -print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") -print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") -# add printing 10 percentiles of latency to better understand latency distribution -percentiles = [10, 25, 50, 75, 90, 95, 99] -for p in percentiles: - print(f"{p}th percentile latency: {np.percentile(benchmark_results['latencies'], p)*1000:.2f} ms") -print(f"Average document length: {num_tokens / len(docs)} tokens") + #print(benchmark_results) + print("Tokens:",num_tokens) + print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") + print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") + print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") + print(f"Average document length: {num_tokens / len(docs)} tokens") diff --git a/demos/common/export_models/export_model.py b/demos/common/export_models/export_model.py index 9192b91449..b5c1acb2d7 100644 --- a/demos/common/export_models/export_model.py +++ b/demos/common/export_models/export_model.py @@ -92,7 +92,7 @@ def add_common_arguments(parser): parser_text2speech = subparsers.add_parser('text2speech', help='export model for text2speech endpoint') add_common_arguments(parser_text2speech) parser_text2speech.add_argument('--num_streams', default=0, type=int, help='The number of parallel execution streams to use for the models in the pipeline.', dest='num_streams') -parser_text2speech.add_argument('--model_type', default='speecht5', choices=['speecht5', 'kokoro'], help='Type of the source TTS model. speecht5 uses optimum-cli; kokoro uses a dedicated PyTorch->OpenVINO conversion path.', dest='model_type') +parser_text2speech.add_argument('--model_type', default='kokoro', choices=['speecht5', 'kokoro'], help='Type of the source TTS model. speecht5 uses optimum-cli; kokoro uses a dedicated PyTorch->OpenVINO conversion path.', dest='model_type') parser_text2speech.add_argument('--vocoder', type=str, help='The vocoder model to use for speecht5. For example microsoft/speecht5_hifigan. Ignored for kokoro.', dest='vocoder') parser_text2speech.add_argument('--speaker_name', type=str, help='Name of the speaker (speecht5 only; for kokoro all voices from the HF repo are exported).', dest='speaker_name') parser_text2speech.add_argument('--speaker_path', type=str, help='Path to the speaker.bin file (speecht5 only; for kokoro all voices from the HF repo are exported).', dest='speaker_path') @@ -503,15 +503,6 @@ def export_embeddings_model_ov(model_repository_path, source_model, model_name, print("Created graph {}".format(os.path.join(model_repository_path, model_name, 'graph.pbtxt'))) add_servable_to_config(config_file_path, model_name, os.path.relpath(os.path.join(model_repository_path, model_name), os.path.dirname(config_file_path))) -def _list_kokoro_voices(destination_path): - """optimum-cli's Kokoro exporter writes per-voice speaker embeddings to - /voices/.bin. Return the sorted list of voice names.""" - voices_dir = os.path.join(destination_path, "voices") - if not os.path.isdir(voices_dir): - print("Warning: no voices/ directory found under", destination_path) - return [] - return sorted(Path(p).stem for p in Path(voices_dir).glob("*.bin")) - def export_text2speech_model(model_repository_path, source_model, model_name, precision, task_parameters, config_file_path): destination_path = os.path.join(model_repository_path, model_name) print("Exporting text2speech model to ",destination_path) @@ -519,16 +510,12 @@ def export_text2speech_model(model_repository_path, source_model, model_name, pr if model_type == 'kokoro': # optimum-intel registers Kokoro under library_name=kokoro / task=text-to-audio. - # The kokoro exporter also dumps each speaker embedding to voices/.bin. if not os.path.isfile(os.path.join(destination_path, 'openvino_model.xml')) or args['overwrite_models']: optimum_command = "optimum-cli export openvino --model {} --task text-to-audio --weight-format {} {} --trust-remote-code {}".format( source_model, precision, task_parameters['extra_quantization_params'], destination_path) print('Running command:', optimum_command) if os.system(optimum_command): raise ValueError("Failed to export kokoro model", source_model) - voice_names = _list_kokoro_voices(destination_path) - # Render the graph with every available voice (path is relative to graph.pbtxt). - task_parameters['voices'] = [{'name': n, 'path': f'./voices/{n}.bin'} for n in voice_names] else: if not os.path.isdir(destination_path) or args['overwrite_models']: if not task_parameters.get('vocoder'): diff --git a/distro.bzl b/distro.bzl index c01ad7bb56..a5f43dfd36 100644 --- a/distro.bzl +++ b/distro.bzl @@ -43,28 +43,4 @@ def distro_flag(): negate = ":ubuntu_build", ) -# Controls whether espeak-ng is built from source (via Bazel) and bundled -# into the OVMS release. When "off", no espeak-ng artifacts are produced -# and the runtime will not have phonemization fallback available. -def espeak_flag(): - string_flag( - name = "espeak", - values = ["on", "off"], - build_setting_default = "on", - ) - native.config_setting( - name = "espeak_on", - flag_values = { - "espeak": "on", - }, - ) - native.config_setting( - name = "espeak_off", - flag_values = { - "espeak": "off", - }, - ) - more_selects.config_setting_negation( - name = "not_espeak_on", - negate = ":espeak_on", - ) + diff --git a/docs/build_from_source.md b/docs/build_from_source.md index 1b3c9f8579..159792787b 100644 --- a/docs/build_from_source.md +++ b/docs/build_from_source.md @@ -156,6 +156,32 @@ dist/ubuntu22 └── ovms.tar.gz.sha256 ``` +### Optional `espeak-ng` for Speech Generation + +`espeak-ng` is optional in OVMS builds. It is used by Kokoro speech generation for: + +- non-English grapheme-to-phoneme conversion, +- fallback phonemization of some out-of-vocabulary (OOV) English words. + +If you do not need Kokoro non-English support and can accept reduced English OOV fallback behavior, you can: + +- skip building `espeak-ng` during image/package build: + +```bash +make targz_package ESPEAK=0 +``` + +- or remove it from an already prepared package by deleting: + - `libespeak-ng.so*` from the package `lib` directory + - `share/espeak-ng-data/` + +Consequences of removing or disabling `espeak-ng`: + +- Speech generation endpoint still works for Kokoro English usage. +- Non-English Kokoro text normalization/phonemization paths that depend on `espeak-ng` are not available. +- English OOV words no longer use `espeak-ng` fallback, so pronunciation quality/coverage for uncommon words may degrade. +- In practice, treat Kokoro as English-focused (for example `en-us` and `en-gb`) when `espeak-ng` is not present. + --- Read more details about building and testing changes in [developer guide](./developer_guide.md). diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 0bb0257580..3d57f7b91e 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -27,7 +27,7 @@ LEGACY_MODEL_FILE="1/model.bin" EMBEDDING_MODEL="thenlper/gte-small" RERANK_MODEL="BAAI/bge-reranker-base" VLM_MODEL="OpenVINO/InternVL2-1B-int4-ov" -TTS_MODEL="microsoft/speecht5_tts" +TTS_MODEL="hexgrad/Kokoro-82M" STT_MODEL="openai/whisper-tiny" # Models for tools testing. Only tokenizers are downloaded. @@ -48,7 +48,7 @@ echo "Downloading LLM testing models to directory $1" export PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://storage.openvinotoolkit.org/simple/wheels/nightly" if [ "$2" = "docker" ]; then export PATH=$PATH:/opt/intel/openvino/python/bin - python3 -m pip install "optimum-intel"@git+https://github.com/huggingface/optimum-intel.git nncf sentence_transformers==5.3.0 sentencepiece requests protobuf==7.35.0 + python3 -m pip install "optimum-intel"@git+https://github.com/huggingface/optimum-intel.git nncf sentence_transformers==5.3.0 sentencepiece requests protobuf==7.35.0 kokoro else python3 -m venv .venv . .venv/bin/activate @@ -83,13 +83,13 @@ if [ ! -f "$1/$FACEBOOK_MODEL/chat_template.jinja" ]; then cp src/test/llm/dummy_facebook_template.jinja "$1/$FACEBOOK_MODEL/chat_template.jinja" fi -if [ -f "$1/$TTS_MODEL/$TOKENIZER_FILE" ]; then - echo "Model file $1/$TTS_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." +if [ -f "$1/$TTS_MODEL/openvino_model.xml" ]; then + echo "Model file $1/$TTS_MODEL/openvino_model.xml exists. Skipping downloading models." else - python3 demos/common/export_models/export_model.py text2speech --source_model "$TTS_MODEL" --weight-format int4 --model_repository_path $1 --vocoder microsoft/speecht5_hifigan + python3 demos/common/export_models/export_model.py text2speech --source_model "$TTS_MODEL" --model_type kokoro --weight-format int8 --model_repository_path $1 fi -if [ ! -f "$1/$TTS_MODEL/$TOKENIZER_FILE" ]; then - echo "[ERROR] Model file $1/$TTS_MODEL/$TOKENIZER_FILE does not exist." +if [ ! -f "$1/$TTS_MODEL/openvino_model.xml" ]; then + echo "[ERROR] Model file $1/$TTS_MODEL/openvino_model.xml does not exist." exit 1 fi diff --git a/src/BUILD b/src/BUILD index 2f271b28c2..2ef747b99e 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2084,12 +2084,6 @@ cc_binary( "//src/python/binding:pyovms.so", ], "//:disable_python": [] - }) + select({ - "//:espeak_on": [ - "//third_party:espeak_ng", - "//third_party:espeak_ng_data", - ], - "//:espeak_off": [], }), # linkstatic = False, # Use for dynamic linking when necessary ) diff --git a/src/audio/text_to_speech/t2s_servable.cpp b/src/audio/text_to_speech/t2s_servable.cpp index a0af03b2cf..20d838a015 100644 --- a/src/audio/text_to_speech/t2s_servable.cpp +++ b/src/audio/text_to_speech/t2s_servable.cpp @@ -15,10 +15,13 @@ //***************************************************************************** #include +#include #include #include #include #include +#include +#include #include "openvino/genai/speech_generation/text2speech_pipeline.hpp" #include "src/audio/text_to_speech/t2s_calculator.pb.h" @@ -31,6 +34,8 @@ namespace ovms { +static constexpr const char* VOICES_DIR_NAME = "voices"; + static size_t getShapeElementsCount(const ov::Shape& shape) { size_t elementsCount = 1; for (const auto dim : shape) { @@ -42,6 +47,35 @@ static size_t getShapeElementsCount(const ov::Shape& shape) { return elementsCount; } +static std::vector getVoiceEmbeddingPaths(const std::filesystem::path& voicesDir) { + std::vector voicePaths; + std::error_code errorCode; + std::filesystem::directory_iterator directoryIt(voicesDir, errorCode); + if (errorCode) { + throw std::runtime_error("Failed to open voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + + const std::filesystem::directory_iterator directoryEnd; + for (; directoryIt != directoryEnd; directoryIt.increment(errorCode)) { + if (errorCode) { + throw std::runtime_error("Failed to iterate voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + const auto& entry = *directoryIt; + const bool isRegularFile = entry.is_regular_file(errorCode); + if (errorCode) { + throw std::runtime_error("Failed to inspect entry '" + entry.path().string() + "' in voices directory '" + voicesDir.string() + "': " + errorCode.message()); + } + if (!isRegularFile) { + continue; + } + if (entry.path().extension() == ".bin") { + voicePaths.emplace_back(entry.path()); + } + } + std::sort(voicePaths.begin(), voicePaths.end()); + return voicePaths; +} + static ov::Tensor readSpeakerEmbedding(const std::filesystem::path& filePath, const ov::Shape& expectedShape) { std::ifstream input(filePath, std::ios::binary); if (input.fail()) { @@ -100,6 +134,8 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD device = recommendTargetDevice(); SPDLOG_INFO("No device specified for TTS model, using recommended device: {}", device); } + // Normalize the path to use OS-appropriate separators + parsedModelsPath = std::filesystem::absolute(parsedModelsPath); ov::AnyMap config; Status status = JsonParser::parsePluginConfig(pluginConfig, config); if (!status.ok()) { @@ -108,6 +144,21 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD } ttsPipeline = std::make_shared(parsedModelsPath.string(), device, config); const ov::Shape speakerEmbeddingShape = ttsPipeline->get_speaker_embedding_shape(); + const std::filesystem::path voicesDir = parsedModelsPath / VOICES_DIR_NAME; + std::error_code ec; + if (std::filesystem::is_directory(voicesDir, ec)) { + try { + for (const auto& voicePath : getVoiceEmbeddingPaths(voicesDir)) { + const std::string voiceName = voicePath.stem().string(); + voices.insert_or_assign(voiceName, readSpeakerEmbedding(voicePath, speakerEmbeddingShape)); + } + } catch (const std::exception& e) { + SPDLOG_WARN("Failed to load voices from {}: {}", voicesDir.string(), e.what()); + } + } else { + SPDLOG_DEBUG("Voices directory not found: {}", voicesDir.string()); + } + for (const auto& voice : graphVoices) { std::filesystem::path voicePath(voice.path()); if (voicePath.is_relative()) { @@ -115,6 +166,9 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD } if (!std::filesystem::exists(voicePath)) throw std::runtime_error{"Requested voice speaker embeddings file does not exist: " + voicePath.string()}; + if (voices.find(voice.name()) != voices.end()) { + SPDLOG_DEBUG("Voice '{}' is already configured and will be overwritten with tensor from: {}", voice.name(), voicePath.string()); + } voices[voice.name()] = readSpeakerEmbedding(voicePath, speakerEmbeddingShape); } } diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index d275a57588..ca2e41ffc1 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -350,23 +350,6 @@ static Status createTextToSpeechGraphTemplate(const std::string& directoryPath, SPDLOG_TRACE("modelsPath: {}, directoryPath: {}, ggufFilename: {}", modelsPath, directoryPath, ggufFilename.value_or("std::nullopt")); GET_PLUGIN_CONFIG_OPT_OR_FAIL_AND_RETURN(exportSettings); - // Enumerate kokoro speaker embeddings dumped by optimum-cli to /voices/*.bin. - std::vector voiceNames; - if (exportSettings.modelType == "kokoro") { - std::filesystem::path voicesDir = std::filesystem::path(directoryPath) / "voices"; - std::error_code ec; - if (std::filesystem::is_directory(voicesDir, ec)) { - for (const auto& entry : std::filesystem::directory_iterator(voicesDir, ec)) { - if (entry.is_regular_file(ec) && !ec && entry.path().extension() == ".bin") { - voiceNames.push_back(entry.path().stem().string()); - } - } - std::sort(voiceNames.begin(), voiceNames.end()); - } else { - SPDLOG_WARN("Kokoro voices directory not found at {}", voicesDir.string()); - } - } - // clang-format off oss << R"( input_stream: "HTTP_REQUEST_PAYLOAD:input" @@ -391,19 +374,6 @@ node { oss << R"(plugin_config: ')" << pluginConfigOpt.value() << R"(' )"; } - if (!voiceNames.empty()) { - oss << R"(voices: [)"; - for (size_t i = 0; i < voiceNames.size(); ++i) { - oss << R"( - { name: ")" << voiceNames[i] << R"(", path: "./voices/)" << voiceNames[i] << R"(.bin" })"; - if (i + 1 < voiceNames.size()) { - oss << ","; - } - } - oss << R"( - ] - )"; - } oss << R"(} } })"; diff --git a/src/test/audio/graph_tts.pbtxt b/src/test/audio/graph_tts.pbtxt index 021c525225..71c35b7d9c 100644 --- a/src/test/audio/graph_tts.pbtxt +++ b/src/test/audio/graph_tts.pbtxt @@ -24,15 +24,9 @@ node { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" - voices: [ - { - name: "speaker1", - path: "/ovms/src/test/audio/speaker.bin", - } - ] } } } \ No newline at end of file diff --git a/src/test/audio/text2speech_test.cpp b/src/test/audio/text2speech_test.cpp index f4d5a88ac1..98066a95c7 100644 --- a/src/test/audio/text2speech_test.cpp +++ b/src/test/audio/text2speech_test.cpp @@ -21,7 +21,6 @@ #include "../../audio/audio_utils.hpp" #include "../../http_rest_api_handler.hpp" #include "../../server.hpp" -#include "rapidjson/document.h" #include "../test_http_utils.hpp" #include "../test_utils.hpp" #include "../platform_utils.hpp" @@ -58,7 +57,8 @@ TEST_F(Text2SpeechHttpTest, simplePositive) { { "model": ")" + modelName + R"(", - "input": "The quick brown fox jumped over the lazy dog." + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -74,7 +74,8 @@ TEST_F(Text2SpeechHttpTest, emptyInput) { { "model": ")" + modelName + R"(", - "input": "" + "input": "", + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -103,7 +104,7 @@ TEST_F(Text2SpeechHttpTest, positiveWithVoice) { "model": ")" + modelName + R"(", "input": "The quick brown fox jumped over the lazy dog.", - "voice": "speaker1" + "voice": "af_alloy" } )"; ASSERT_EQ( @@ -143,7 +144,7 @@ TEST_F(Text2SpeechConfigTest, NodeNameMissing) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" target_device: "CPU" } } @@ -169,7 +170,7 @@ TEST_F(Text2SpeechConfigTest, SidePacketMissing) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" target_device: "CPU" } } @@ -222,7 +223,7 @@ TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: 'INVALID', target_device: "CPU" } @@ -236,6 +237,34 @@ TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } +TEST_F(Text2SpeechConfigTest, MissingVoicesInGraphUsesModelVoicesDir) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node { + name: "ttsNode1" + input_side_packet: "TTS_NODE_RESOURCES:t2s_servable" + calculator: "T2sCalculator" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + node_options: { + [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" + plugin_config: '{"NUM_STREAMS": "1" }', + target_device: "CPU" + } + } + } + )"; + + ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; + DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); + mediapipeDummy.inputConfig = testPbtxt; + ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::OK); +} + TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { ConstructorEnabledModelManager manager; std::string testPbtxt = R"( @@ -250,7 +279,7 @@ TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ @@ -284,7 +313,7 @@ TEST_F(Text2SpeechConfigTest, VoiceMissingPath) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ @@ -317,7 +346,7 @@ TEST_F(Text2SpeechConfigTest, VoiceInvalidFile) { output_stream: "HTTP_RESPONSE_PAYLOAD:output" node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { - models_path: "/ovms/src/test/llm_testing/microsoft/speecht5_tts" + models_path: "/ovms/src/test/llm_testing/hexgrad/Kokoro-82M" plugin_config: '{"NUM_STREAMS": "1" }', target_device: "CPU" voices: [ diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index f0caa528fe..750e5558a4 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -809,12 +809,6 @@ TEST_F(GraphCreationTest, textToSpeechPositiveDefault) { } TEST_F(GraphCreationTest, textToSpeechPositiveKokoro) { - // Pre-create the voices/ directory that optimum-cli would have populated for kokoro. - std::filesystem::path voicesDir = std::filesystem::path(this->directoryPath) / "voices"; - std::filesystem::create_directories(voicesDir); - { std::ofstream f(voicesDir / "af_alloy.bin"); } - { std::ofstream f(voicesDir / "am_adam.bin"); } - ovms::HFSettingsImpl hfSettings; hfSettings.task = ovms::TEXT_TO_SPEECH_GRAPH; hfSettings.exportSettings.modelName = "myModel"; diff --git a/third_party/BUILD b/third_party/BUILD index bce804d7a1..2622baa0a9 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -61,27 +61,16 @@ alias( ) # espeak-ng built from source via Bazel (rules_foreign_cc cmake). -# Selected on/off via the //:espeak build flag. When disabled this resolves -# to an empty cc_library so dependents can unconditionally list it. -cc_library( - name = "espeak_ng_empty", - visibility = ["//visibility:public"], -) - +# Built as a separate step in the Dockerfile (ARG ESPEAK=1/0), not as +# a dependency of the OVMS binary. alias( name = "espeak_ng", - actual = select({ - "//:espeak_on": "@espeak_ng//:espeak_ng", - "//:espeak_off": ":espeak_ng_empty", - }), + actual = "@espeak_ng//:espeak_ng", visibility = ["//visibility:public"], ) alias( name = "espeak_ng_data", - actual = select({ - "//:espeak_on": "@espeak_ng//:espeak_ng_data", - "//:espeak_off": ":espeak_ng_empty", - }), + actual = "@espeak_ng//:espeak_ng_data", visibility = ["//visibility:public"], ) \ No newline at end of file diff --git a/windows_build.bat b/windows_build.bat index eac54243e7..4c3f100075 100644 --- a/windows_build.bat +++ b/windows_build.bat @@ -49,18 +49,10 @@ IF "%~4"=="--integrity" ( set "buildWithIntegrity=" ) -:: Allow disabling espeak-ng (built from source via Bazel) by setting -:: ESPEAK=0 before invoking this script. Defaults to on. -IF "%ESPEAK%"=="0" ( - set "espeakArg=--//:espeak=off" -) ELSE ( - set "espeakArg=--//:espeak=on" -) - set "bazelStartupCmd=--output_user_root=!BAZEL_SHORT_PATH!" set "openvino_dir=!BAZEL_SHORT_PATH!/openvino/runtime/cmake" -set "buildCommand=bazel %bazelStartupCmd% build %buildWithIntegrity% %bazelBuildArgs% %espeakArg% --action_env OpenVINO_DIR=%openvino_dir% --jobs=%NUMBER_OF_PROCESSORS% --verbose_failures %buildTargets% 2>&1 | tee win_build.log" +set "buildCommand=bazel %bazelStartupCmd% build %buildWithIntegrity% %bazelBuildArgs% --action_env OpenVINO_DIR=%openvino_dir% --jobs=%NUMBER_OF_PROCESSORS% --verbose_failures %buildTargets% 2>&1 | tee win_build.log" set "setOvmsVersionCmd=python windows_set_ovms_version.py" :: Setting PATH environment variable based on default windows node settings: Added ovms_windows specific python settings and c:/opt and removed unused Nvidia and OCL specific tools. diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index 48f13e61b8..72cab81c98 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -33,7 +33,7 @@ set "RERANK_MODEL=BAAI/bge-reranker-base" set "TEXT_GENERATION_MODEL=HuggingFaceTB/SmolLM2-360M-Instruct" set "FACEBOOK_MODEL=facebook/opt-125m" set "VLM_MODEL=OpenVINO/InternVL2-1B-int4-ov" -set "TTS_MODEL=microsoft/speecht5_tts" +set "TTS_MODEL=hexgrad/Kokoro-82M" set "STT_MODEL=openai/whisper-tiny" :: Models for tools testing. Only tokenizers are downloaded. @@ -64,7 +64,7 @@ if not exist "%~1" mkdir "%~1" :: Export models -call :download_export_model_tts "%TTS_MODEL%" "text2speech" "--weight-format int4" "%~1" +call :download_export_model_tts "%TTS_MODEL%" "text2speech" "--model_type kokoro --weight-format int8" "%~1" call :download_export_model "%STT_MODEL%" "speech2text" "--weight-format int4" "%~1" call :download_openvino "%VLM_MODEL%" "%~1" OpenGVLab/InternVL2-1B call :download_export_model "%TEXT_GENERATION_MODEL%" "text_generation" "--weight-format int8" "%~1" @@ -113,11 +113,11 @@ set "model_type=%~2" set "export_args=%~3" set "repository=%~4" -if not exist "%repository%\%model%\openvino_tokenizer.bin" ( +if not exist "%repository%\%model%\openvino_model.xml" ( echo Downloading %model_type% model to %repository%\%model% directory. - python demos\common\export_models\export_model.py %model_type% --source_model "%model%" %export_args% --vocoder microsoft/speecht5_hifigan --model_repository_path %repository% + python demos\common\export_models\export_model.py %model_type% --source_model "%model%" %export_args% --model_repository_path %repository% ) else ( - echo Models file %repository%\%model%\openvino_tokenizer.bin exists. Skipping downloading models. + echo Models file %repository%\%model%\openvino_model.xml exists. Skipping downloading models. ) exit /b 0 From 3ab4b0421a3a90c15668a57e9423572d18fd69d6 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 14:06:49 +0200 Subject: [PATCH 02/11] fix --- src/graph_export/graph_export.cpp | 7 ------- src/test/graph_export_test.cpp | 4 ---- 2 files changed, 11 deletions(-) diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index ca2e41ffc1..f1ee7adb2d 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -366,13 +366,6 @@ node { models_path: ")" << modelsPath << R"(" )"; - if (!exportSettings.targetDevice.empty()) { - oss << R"(target_device: ")" << exportSettings.targetDevice << R"(" - )"; - } - if (pluginConfigOpt.has_value()) { - oss << R"(plugin_config: ')" << pluginConfigOpt.value() << R"(' - )"; } oss << R"(} } diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index 750e5558a4..364765585e 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -420,10 +420,6 @@ node { node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { models_path: "./" - voices: [ - { name: "af_alloy", path: "./voices/af_alloy.bin" }, - { name: "am_adam", path: "./voices/am_adam.bin" } - ] } } } From 7e086455defe4d5ce46db8caf8cf35922a7388cd Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 14:16:45 +0200 Subject: [PATCH 03/11] style --- src/graph_export/graph_export.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index f1ee7adb2d..78fe63ee0e 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -379,14 +379,14 @@ node { return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; } #endif - // clang-format on - if (!writeToFile) { - inMemoryGraphContent = oss.str(); - return StatusCode::OK; - } - std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); - return FileSystem::createFileOverwrite(fullPath, oss.str()); +// clang-format on +if (!writeToFile) { + inMemoryGraphContent = oss.str(); + return StatusCode::OK; } +std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); +return FileSystem::createFileOverwrite(fullPath, oss.str()); +} // namespace ovms static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { From 052c200cc7ea3d138450ea6651e491ac4383e528 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 14:18:37 +0200 Subject: [PATCH 04/11] style --- src/graph_export/graph_export.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index 78fe63ee0e..88056d0e84 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -758,5 +758,4 @@ std::variant, Status> GraphExport::createPluginString return std::nullopt; } } - } // namespace ovms From 8ef6ad003a038b647b0b9486b93c322668258fde Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 14:36:54 +0200 Subject: [PATCH 05/11] fix --- src/graph_export/graph_export.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index 88056d0e84..1584c07387 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -366,6 +366,13 @@ node { models_path: ")" << modelsPath << R"(" )"; + if (!exportSettings.targetDevice.empty()) { + oss << R"(target_device: ")" << exportSettings.targetDevice << R"(" + )"; + } + if (pluginConfigOpt.has_value()) { + oss << R"(plugin_config: ')" << pluginConfigOpt.value() << R"(' + )"; } oss << R"(} } @@ -379,14 +386,14 @@ node { return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; } #endif -// clang-format on -if (!writeToFile) { - inMemoryGraphContent = oss.str(); - return StatusCode::OK; + // clang-format on + if (!writeToFile) { + inMemoryGraphContent = oss.str(); + return StatusCode::OK; + } + std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); + return FileSystem::createFileOverwrite(fullPath, oss.str()); } -std::string fullPath = FileSystem::joinPath({directoryPath, "graph.pbtxt"}); -return FileSystem::createFileOverwrite(fullPath, oss.str()); -} // namespace ovms static Status createSpeechToTextGraphTemplate(const std::string& directoryPath, const HFSettingsImpl& hfSettings, bool writeToFile) { if (!std::holds_alternative(hfSettings.graphSettings)) { From 9f335c87a533aa6f697e0611cb45cf30ce1546a6 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 15:50:39 +0200 Subject: [PATCH 06/11] fix --- demos/audio/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/audio/README.md b/demos/audio/README.md index 07a4333668..8bfdcd604c 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -86,7 +86,7 @@ An asynchronous benchmarking client can be used to access the model server perfo > Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. ```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt openai>=1.0.0 curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model Kokoro-82M-OpenVINO-FP16-OVMS --batch_size 1 --limit 1000 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --voice af_alloy Number of documents: 1000 From 1583c904c7283297cc864bbc68621ccf378be4b0 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 19:53:29 +0200 Subject: [PATCH 07/11] fix --- demos/audio/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/audio/README.md b/demos/audio/README.md index 8bfdcd604c..79aa26b78c 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -257,7 +257,7 @@ mkdir -p models Export Speech-to-Text model with word timestamps enabled: ```console -python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps +python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps --task speech2text ``` :::{dropdown} **Deploying with Docker** From 0c24a446352fc97ad660100bfe7251efd1eca06f Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Thu, 16 Jul 2026 22:58:37 +0200 Subject: [PATCH 08/11] fix --- demos/audio/README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/demos/audio/README.md b/demos/audio/README.md index 79aa26b78c..bdcd89c14c 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -115,7 +115,7 @@ Select deployment option depending on how you prepared models in the previous st Running this command starts the container with CPU only target device: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path /models +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device CPU ``` **GPU** @@ -124,7 +124,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device GPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device GPU ``` ::: @@ -133,7 +133,7 @@ docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-a If you run on GPU make sure to have appropriate drivers installed, so the device is accessible for the model server. ```bat -ovms --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path models --target_device GPU +ovms --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path models --target_device GPU ``` ::: @@ -148,7 +148,7 @@ Transcribe the speech.wav file generated in the [Speech generation](#speech-gene ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" -F language="en" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-fp16-ov" -F language="en" ``` ```json {"text": " The quick brown fox jumped over the lazy dog."} @@ -170,7 +170,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="OpenVINO/whisper-large-v3-turbo-fp16-ov", + model="whisper-large-v3-turbo-fp16-ov", language="en", file=audio_file ) @@ -188,7 +188,7 @@ The quick brown fox jumped over the lazy dog. curl -N http://localhost:8000/v3/audio/transcriptions \ -H "Content-Type: multipart/form-data" \ -F file="@speech.wav" \ - -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" \ + -F model="whisper-large-v3-turbo-fp16-ov" \ -F language="en" \ -F stream="true" ``` @@ -207,7 +207,7 @@ data: {"type":"transcript.text.done","text":"The quick brown fox jumped over the ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" -F language="en" -F timestamp_granularities[]="segment" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-fp16-ov" -F language="en" -F timestamp_granularities[]="segment" ``` ```json {"text":" A quick brown fox jumped over the lazy dog","segments":[{"text":" A quick brown fox jumped over the lazy dog","start":0.0,"end":3.1399998664855957}]} @@ -229,7 +229,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="OpenVINO/whisper-large-v3-turbo-fp16-ov", + model="whisper-large-v3-turbo-fp16-ov", language="en", response_format="verbose_json", timestamp_granularities=["segment"], @@ -257,20 +257,20 @@ mkdir -p models Export Speech-to-Text model with word timestamps enabled: ```console -python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps --task speech2text +python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps ``` :::{dropdown} **Deploying with Docker** ```bash -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts --task speech2text ``` ::: :::{dropdown} **Deploying on Bare Metal** ```bat -ovms --rest_port 8000 --model_path models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts +ovms --rest_port 8000 --model_path models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts --task speech2text ``` ::: @@ -342,7 +342,7 @@ pip install -r requirements/requirements.txt -r requirements/requirements-api.tx Run evaluation example: ```console PYTHONPATH=. python api/run_eval.py \ - --model_name openai/OpenVINO/whisper-large-v3-turbo-fp16-ov \ + --model_name openai/whisper-large-v3-turbo-fp16-ov \ --dataset_path "hf-audio/esb-datasets-test-only-sorted" \ --max_workers 1 \ --split test.clean \ @@ -365,8 +365,8 @@ Where: - WER (Word Error Rate) is the percentage of transcription errors compared to the reference text (substitutions + deletions + insertions). Lower is better. - RTFx (Real-Time Factor, expressed as speedup) indicates processing speed relative to audio duration. Values above 1 mean faster-than-real-time transcription (for example, 5.16 means about 5.16x real time). -**For Open ASR Leaderboard, run `run_eval.py` with model name prefixed by `openai/` (for example `openai/OpenVINO/whisper-large-v3-turbo-fp16-ov`).** -**OVMS should still be deployed with `--model_name OpenVINO/whisper-large-v3-turbo-fp16-ov` (evaluation script does not include `openai/` prefix in requests).** +**For Open ASR Leaderboard, run `run_eval.py` with model name prefixed by `openai/` (for example `openai/whisper-large-v3-turbo-fp16-ov`).** +**OVMS should still be deployed with `--model_name whisper-large-v3-turbo-fp16-ov` (evaluation script does not include `openai/` prefix in requests).** You can replace `librispeech` with other datasets supported by the leaderboard configuration. For multilingual models run_eval_ml.py should be used. ## Translation @@ -389,7 +389,7 @@ Select deployment option depending on how you prepared models in the previous st Running this command starts the container with CPU only target device: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device CPU ``` **GPU** @@ -398,7 +398,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device GPU ``` ::: @@ -408,11 +408,11 @@ If you run on GPU make sure to have appropriate drivers installed, so the device ```bat mkdir models -ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device CPU +ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device CPU ``` or ```bat -ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU +ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device GPU ``` ::: @@ -423,7 +423,7 @@ Translate the speech_spanish.wav file generated above. ```bash -curl http://localhost:8000/v3/audio/translations -H "Content-Type: multipart/form-data" -F file="@speech_spanish.wav" -F model="OpenVINO/whisper-large-v3-fp16-ov" +curl http://localhost:8000/v3/audio/translations -H "Content-Type: multipart/form-data" -F file="@speech_spanish.wav" -F model="whisper-large-v3-fp16-ov" ``` ```json {"text": " Madrid is the capital of Spain."} @@ -445,7 +445,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") translation = client.audio.translations.create( - model="OpenVINO/whisper-large-v3-fp16-ov", + model="whisper-large-v3-fp16-ov", file=audio_file ) From 61470e06278ad0a0f2f9f5796e426c465514af48 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Fri, 17 Jul 2026 10:30:35 +0200 Subject: [PATCH 09/11] fix --- src/test/audio/text2speech_test.cpp | 50 +++++++++++------------------ 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/src/test/audio/text2speech_test.cpp b/src/test/audio/text2speech_test.cpp index 98066a95c7..9313eab8ca 100644 --- a/src/test/audio/text2speech_test.cpp +++ b/src/test/audio/text2speech_test.cpp @@ -131,6 +131,16 @@ TEST_F(Text2SpeechHttpTest, nonExistingVoiceRequested) { class Text2SpeechConfigTest : public ::testing::Test {}; +namespace { +ovms::Status validateText2SpeechGraphConfig(ConstructorEnabledModelManager& manager, std::string testPbtxt) { + adjustConfigForTargetPlatform(testPbtxt); + ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; + DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); + mediapipeDummy.inputConfig = testPbtxt; + return mediapipeDummy.validate(manager); +} +} // namespace + TEST_F(Text2SpeechConfigTest, NodeNameMissing) { ConstructorEnabledModelManager manager; std::string testPbtxt = R"( @@ -151,10 +161,7 @@ TEST_F(Text2SpeechConfigTest, NodeNameMissing) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::LLM_NODE_MISSING_NAME); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::LLM_NODE_MISSING_NAME); } TEST_F(Text2SpeechConfigTest, SidePacketMissing) { @@ -177,10 +184,7 @@ TEST_F(Text2SpeechConfigTest, SidePacketMissing) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_INITIALIZATION_ERROR); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_INITIALIZATION_ERROR); } TEST_F(Text2SpeechConfigTest, MissingModelsPath) { @@ -203,10 +207,7 @@ TEST_F(Text2SpeechConfigTest, MissingModelsPath) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { @@ -231,10 +232,7 @@ TEST_F(Text2SpeechConfigTest, InvalidPluginConfig) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } TEST_F(Text2SpeechConfigTest, MissingVoicesInGraphUsesModelVoicesDir) { @@ -259,10 +257,7 @@ TEST_F(Text2SpeechConfigTest, MissingVoicesInGraphUsesModelVoicesDir) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::OK); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::OK); } TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { @@ -293,10 +288,7 @@ TEST_F(Text2SpeechConfigTest, NonExistingVoicePath) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } TEST_F(Text2SpeechConfigTest, VoiceMissingPath) { @@ -326,10 +318,7 @@ TEST_F(Text2SpeechConfigTest, VoiceMissingPath) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } TEST_F(Text2SpeechConfigTest, VoiceInvalidFile) { @@ -360,8 +349,5 @@ TEST_F(Text2SpeechConfigTest, VoiceInvalidFile) { } )"; - ovms::MediapipeGraphConfig mgc{"mediaDummy", "", ""}; - DummyMediapipeGraphDefinition mediapipeDummy("mediaDummy", mgc, testPbtxt, nullptr); - mediapipeDummy.inputConfig = testPbtxt; - ASSERT_EQ(mediapipeDummy.validate(manager), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); + ASSERT_EQ(validateText2SpeechGraphConfig(manager, testPbtxt), StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID); } From 316d061ffd45b9d51f00b1cf5ec68e58b9fd8bbf Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Fri, 17 Jul 2026 12:40:51 +0200 Subject: [PATCH 10/11] Remove docs changes - will be merged in separate PR --- demos/audio/README.md | 342 ++++++++++++++++---------------- demos/benchmark/v3/benchmark.py | 74 ++----- 2 files changed, 195 insertions(+), 221 deletions(-) diff --git a/demos/audio/README.md b/demos/audio/README.md index bdcd89c14c..8b13a657b1 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -8,46 +8,116 @@ Check supported [Speech Recognition Models](https://openvinotoolkit.github.io/op ## Prerequisites -**OVMS version 2026.3** This demo requires version 2026.3 or nightly release. +**OVMS version 2025.4** This demo require version 2025.4 or nightly release. + +**Model preparation**: Python 3.10 or higher with pip **Model Server deployment**: Installed Docker Engine or OVMS binary package according to the [baremetal deployment guide](../../docs/deploying_server_baremetal.md) **Client**: curl or Python for using OpenAI client package ## Speech generation -Kokoro is the primary example in this demo, but SpeechT5 remains supported for existing deployments. +### Prepare speaker embeddings +When generating speech you can use default speaker voice or you can prepare your own speaker embedding file. Here you can see how to do it with downloaded file from online repository, but you can try with your own speech recording as well: +```console +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/requirements.txt +mkdir audio_samples +curl --create-dirs "https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0032_8k.wav" -o audio_samples/audio.wav +curl --create-dirs https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/audio/create_speaker_embedding.py -o models/speakers/create_speaker_embedding.py +python models/speakers/create_speaker_embedding.py audio_samples/audio.wav models/speakers/voice1.bin +``` + ### Model preparation -This demo uses a pre-exported OpenVINO IR model [luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS](https://huggingface.co/luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS) available on HuggingFace. -The model can be pulled directly by OVMS without any conversion step. +Supported models should use the topology of [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts) which needs to be converted to IR format before using in OVMS. + +Specific OVMS pull mode example for models requiring conversion is described in the [Ovms pull mode](../../docs/pull_hf_models.md#pulling-models-outside-openvino-organization) + +Or you can use the python export_model.py script described below. + +Here, the original Text to Speech model will be converted to IR format and optionally quantized. +That ensures faster initialization time, better performance and lower memory consumption. +Execution parameters will be defined inside the `graph.pbtxt` file. + +Download export script, install it's dependencies and create directory for the models: +```console +curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt +mkdir models +``` + +Run `export_model.py` script to download and quantize the model: + +> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before running the export script to connect to the HF Hub. +> **Note:** Exporting `microsoft/speecht5_tts` model requires Python 3.10 + +**CPU** +```console +python export_model.py text2speech --source_model microsoft/speecht5_tts --weight-format fp16 --model_name microsoft/speecht5_tts --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan --speaker_name voice1 --speaker_path models/speakers/voice1.bin +``` -> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before starting OVMS to connect to the HF Hub. -> **Note:** Kokoro voices are loaded from the `voices/` directory in the model repository. OVMS loads each voice file under the filename without the extension, for example `af_alloy` for `af_alloy.bin`. +> **Note:** Change the `--weight-format` to quantize the model to `int8` precision to reduce memory consumption and improve performance. +> **Note:** `speaker_name` and `speaker_path` may be omitted if the default model voice is sufficient -See the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. +The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [T2s calculator documentation](../../docs/speech_generation/reference.md) to learn more about configuration options and limitations. ### Deployment -**Deploying with Docker** +**CPU** +Running this command starts the container with CPU only target device: ```bash -mkdir models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path /models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU --task text2speech +mkdir -p models +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts ``` **Deploying on Bare Metal** ```bat -mkdir c:\models -ovms --rest_port 8000 --source_model luis-castillo/Kokoro-82M-OpenVINO-FP16-OVMS --model_repository_path c:\models --model_name Kokoro-82M-OpenVINO-FP16-OVMS --target_device CPU --task text2speech +mkdir models +ovms --rest_port 8000 --model_path models/microsoft/speecht5_tts --model_name microsoft/speecht5_tts ``` ### Request Generation +:::{dropdown} **Unary call with curl with default voice** + + +```bash +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav +``` +::: + +:::{dropdown} **Unary call with OpenAI python library with default voice** + +```python +from pathlib import Path +from openai import OpenAI + +prompt = "The quick brown fox jumped over the lazy dog" +filename = "speech.wav" +url="http://localhost:8000/v3" + + +speech_file_path = Path(__file__).parent / "speech.wav" +client = OpenAI(base_url=url, api_key="not_used") + +with client.audio.speech.with_streaming_response.create( + model="microsoft/speecht5_tts", + voice=None, + input=prompt +) as response: + response.stream_to_file(speech_file_path) + + +print("Generation finished") +``` +::: + :::{dropdown} **Unary call with curl** ```bash -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"af_alloy\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"microsoft/speecht5_tts\", \"voice\":\"voice1\", \"input\": \"The quick brown fox jumped over the lazy dog\"}" -o speech.wav ``` ::: @@ -66,8 +136,8 @@ speech_file_path = Path(__file__).parent / "speech.wav" client = OpenAI(base_url=url, api_key="not_used") with client.audio.speech.with_streaming_response.create( - model="Kokoro-82M-OpenVINO-FP16-OVMS", - voice="af_alloy", + model="microsoft/speecht5_tts", + voice="voice1", input=prompt ) as response: response.stream_to_file(speech_file_path) @@ -82,29 +152,54 @@ Play speech.wav file to check generated speech. ## Benchmarking speech generation An asynchronous benchmarking client can be used to access the model server performance with various load conditions. Below are execution examples captured on Intel(R) Core(TM) Ultra 7 258V. -> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. -> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. - ```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt openai>=1.0.0 +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py -python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model Kokoro-82M-OpenVINO-FP16-OVMS --batch_size 1 --limit 1000 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --voice af_alloy -Number of documents: 1000 -100%|█████████████████████████████████████████████████████████████████████████████████| 1000/1000 [16:37<00:00, 1.00it/s] -Success rate: 100.0%. (1000/1000) -Mean latency: 1506.60 ms -Median latency: 1072.72 ms -Mean RTFx: 3.655x -Median RTFx: 3.899x +python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model microsoft/speecht5_tts --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --tokenizer openai/whisper-large-v3-turbo --trust-remote-code True +Number of documents: 100 +100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [01:58<00:00, 1.19s/it] +Tokens: 1802 +Success rate: 100.0%. (100/100) +Throughput - Tokens per second: 15.2 +Mean latency: 63653.98 ms +Median latency: 66736.83 ms +Average document length: 18.02 tokens ``` -> **Note:** `RTFx` (Real-Time Factor, inverted) is calculated as `generated_audio_duration / generation_time`. -> Values greater than `1.0x` mean faster-than-real-time generation, while values below `1.0x` mean slower-than-real-time. - ## Transcription ### Model preparation Many variances of Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. -In this demo we will use OpenVINO/whisper-large-v3-turbo-fp16-ov, which is a fine-tuned version of Whisper large-v3. +However in this demo we will use openai/whisper-large-v3-turbo which needs to be converted to IR format before using in OVMS. + +Here, the original Speech to Text model will be converted to IR format and quantized. +That ensures faster initialization time, better performance and lower memory consumption. +Execution parameters will be defined inside the `graph.pbtxt` file. + +Download export script, install it's dependencies and create directory for the models: +```console +curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt +mkdir models +``` + +Run `export_model.py` script to download and quantize the model: + +> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before running the export script to connect to the HF Hub. + +**CPU** +```console +python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name openai/whisper-large-v3-turbo --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps +``` + +**GPU** +```console +python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name openai/whisper-large-v3-turbo --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps --target_device GPU +``` + +> **Note:** Change the `--weight-format` to quantize the model to `int8` precision to reduce memory consumption and improve performance. +> **Note:** `--enable_word_timestamps` can be omitted if there is no need for word timestamps support. + +### Deployment :::{dropdown} **Deploying with Docker** @@ -115,7 +210,7 @@ Select deployment option depending on how you prepared models in the previous st Running this command starts the container with CPU only target device: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device CPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo ``` **GPU** @@ -124,7 +219,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device GPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo ``` ::: @@ -133,14 +228,14 @@ docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-a If you run on GPU make sure to have appropriate drivers installed, so the device is accessible for the model server. ```bat -ovms --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name whisper-large-v3-turbo-fp16-ov --model_repository_path models --target_device GPU +ovms --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo ``` ::: -The default configuration should work in most cases but the parameters can be tuned via OVMS arguments. See the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. +The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. ### Request Generation -Transcribe the speech.wav file generated in the [Speech generation](#speech-generation) section. +Transcript file that was previously generated with audio/speech endpoint. > **Note:** Streaming responses are supported for `audio/transcriptions`. `audio/translations` does not support streaming. @@ -148,7 +243,7 @@ Transcribe the speech.wav file generated in the [Speech generation](#speech-gene ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-fp16-ov" -F language="en" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="openai/whisper-large-v3-turbo" -F language="en" ``` ```json {"text": " The quick brown fox jumped over the lazy dog."} @@ -170,7 +265,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="whisper-large-v3-turbo-fp16-ov", + model="openai/whisper-large-v3-turbo", language="en", file=audio_file ) @@ -188,7 +283,7 @@ The quick brown fox jumped over the lazy dog. curl -N http://localhost:8000/v3/audio/transcriptions \ -H "Content-Type: multipart/form-data" \ -F file="@speech.wav" \ - -F model="whisper-large-v3-turbo-fp16-ov" \ + -F model="openai/whisper-large-v3-turbo" \ -F language="en" \ -F stream="true" ``` @@ -203,18 +298,18 @@ data: {"type":"transcript.text.done","text":"The quick brown fox jumped over the ``` ::: -:::{dropdown} **Unary call with sentence timestamps** +:::{dropdown} **Unary call with timestamps** ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-fp16-ov" -F language="en" -F timestamp_granularities[]="segment" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="openai/whisper-large-v3-turbo" -F language="en" -F timestamp_granularities[]="segment" -F timestamp_granularities[]="word" ``` ```json -{"text":" A quick brown fox jumped over the lazy dog","segments":[{"text":" A quick brown fox jumped over the lazy dog","start":0.0,"end":3.1399998664855957}]} +{"text":" A quick brown fox jumped over the lazy dog","words":[{"word":" A","start":0.0,"end":0.14000000059604645},{"word":" quick","start":0.14000000059604645,"end":0.3400000035762787},{"word":" brown","start":0.3400000035762787,"end":0.7799999713897705},{"word":" fox","start":0.7799999713897705,"end":1.3199999332427979},{"word":" jumped","start":1.3199999332427979,"end":1.7799999713897705},{"word":" over","start":1.7799999713897705,"end":2.0799999237060547},{"word":" the","start":2.0799999237060547,"end":2.259999990463257},{"word":" lazy","start":2.259999990463257,"end":2.5399999618530273},{"word":" dog","start":2.5399999618530273,"end":2.919999837875366}],"segments":[{"text":" A quick brown fox jumped over the lazy dog","start":0.0,"end":3.1399998664855957}]} ``` ::: -:::{dropdown} **Unary call with python OpenAI library with sentence timestamps** +:::{dropdown} **Unary call with python OpenAI library with timestamps** ```python from pathlib import Path @@ -229,154 +324,69 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="whisper-large-v3-turbo-fp16-ov", + model="openai/whisper-large-v3-turbo", language="en", response_format="verbose_json", - timestamp_granularities=["segment"], + timestamp_granularities=["segment", "word"], file=audio_file ) print(transcript.text) print(transcript.segments) +print(transcript.words) ``` ``` A quick brown fox jumped over the lazy dog [TranscriptionSegment(id=None, avg_logprob=None, compression_ratio=None, end=3.1399998664855957, no_speech_prob=None, seek=None, start=0.0, temperature=None, text=' A quick brown fox jumped over the lazy dog', tokens=None)] +[TranscriptionWord(end=0.14000000059604645, start=0.0, word=' A'), TranscriptionWord(end=0.3400000035762787, start=0.14000000059604645, word=' quick'), TranscriptionWord(end=0.7799999713897705, start=0.3400000035762787, word=' brown'), TranscriptionWord(end=1.3199999332427979, start=0.7799999713897705, word=' fox'), TranscriptionWord(end=1.7799999713897705, start=1.3199999332427979, word=' jumped'), TranscriptionWord(end=2.0799999237060547, start=1.7799999713897705, word=' over'), TranscriptionWord(end=2.259999990463257, start=2.0799999237060547, word=' the'), TranscriptionWord(end=2.5399999618530273, start=2.259999990463257, word=' lazy'), TranscriptionWord(end=2.919999837875366, start=2.5399999618530273, word=' dog')] ``` ::: -### Word timestamps -If you need word-level timestamps support, export the model with `export_model.py` and enable this feature during export. - -Prepare export script and dependencies: -```console -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt -mkdir -p models -``` +## Benchmarking transcription +An asynchronous benchmarking client can be used to access the model server performance with various load conditions. Below are execution examples captured on Intel(R) Core(TM) Ultra 7 258V. -Export Speech-to-Text model with word timestamps enabled: ```console -python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps -``` - -:::{dropdown} **Deploying with Docker** - -```bash -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts --task speech2text +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt +curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py +python benchmark.py --api_url http://localhost:8000/v3/audio/transcriptions --model openai/whisper-large-v3-turbo --batch_size 1 --limit 1000 --request_rate inf --dataset edinburghcstr/ami --hf-subset ihm --backend speech2text --trust-remote-code True +Number of documents: 1000 +100%|██████████████████████████████████████████████████████████████████████████████| 1000/1000 [04:44<00:00, 3.51it/s] +Tokens: 10948 +Success rate: 100.0%. (1000/1000) +Throughput - Tokens per second: 38.5 +Mean latency: 26670.64 ms +Median latency: 20772.09 ms +Average document length: 10.948 tokens ``` -::: -:::{dropdown} **Deploying on Bare Metal** - -```bat -ovms --rest_port 8000 --model_path models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts --task speech2text -``` -::: +## Translation +To test translations endpoint we first need to prepare audio file with speech in language other than English, e.g. Spanish. To generate such sample we will use finetuned version of microsoft/speecht5_tts model. -:::{dropdown} **Unary call with cURL (word timestamps)** +**Deploying with Docker** ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-word-ts" -F language="en" -F timestamp_granularities[]="word" -``` - -Example response: -```json -{"text":" A quick brown fox jumped over the lazy dog","words":[{"word":" A","start":0.0,"end":0.14000000059604645},{"word":" quick","start":0.14000000059604645,"end":0.3400000035762787},{"word":" brown","start":0.3400000035762787,"end":0.7799999713897705},{"word":" fox","start":0.7799999713897705,"end":1.3199999332427979},{"word":" jumped","start":1.3199999332427979,"end":1.7799999713897705},{"word":" over","start":1.7799999713897705,"end":2.0799999237060547},{"word":" the","start":2.0799999237060547,"end":2.259999990463257},{"word":" lazy","start":2.259999990463257,"end":2.5399999618530273},{"word":" dog","start":2.5399999618530273,"end":2.919999837875366}]} -``` -::: - -:::{dropdown} **Unary call with python OpenAI library (word timestamps)** - -```python -from pathlib import Path -from openai import OpenAI - -filename = "speech.wav" -url="http://localhost:8000/v3" - - -speech_file_path = Path(__file__).parent / filename -client = OpenAI(base_url=url, api_key="not_used") - -audio_file = open(filename, "rb") -transcript = client.audio.transcriptions.create( - model="whisper-large-v3-turbo-word-ts", - language="en", - response_format="verbose_json", - timestamp_granularities=["word"], - file=audio_file -) - -print(transcript.text) -print(transcript.words) -``` -::: - -## Evaluate transcription accuracy and performance with Open ASR Leaderboard - -You can evaluate model accuracy (for example WER/CER) against ASR datasets using the Open ASR Leaderboard tooling. - -Clone the repository: -```console -git clone https://github.com/huggingface/open_asr_leaderboard.git -cd open_asr_leaderboard -``` +mkdir -p models -Download and apply OVMS API compatibility patch: +python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan - curl -L https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/external/open_asr_leaderboard.patch -o ovms_open_asr_leaderboard.patch - git apply ovms_open_asr_leaderboard.patch +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/speecht5_tts_spanish --model_name speecht5_tts_spanish -Set OpenAI-compatible endpoint variables for OVMS: -```console -export OPENAI_BASE_URL=http://localhost:8000/v3 -export OPENAI_API_KEY="unused" +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav ``` -Install dependencies: -```console -pip install -r requirements/requirements.txt -r requirements/requirements-api.txt openai>=1.0.0 torchcodec==0.12 -``` - -Run evaluation example: -```console -PYTHONPATH=. python api/run_eval.py \ - --model_name openai/whisper-large-v3-turbo-fp16-ov \ - --dataset_path "hf-audio/esb-datasets-test-only-sorted" \ - --max_workers 1 \ - --split test.clean \ - --dataset "librispeech" -``` -Results: -```console -... -Transcribing: 100%|█████████▉| 2617/2620 [12:15<00:00, 5.23it/s] -Transcribing: 100%|█████████▉| 2618/2620 [12:15<00:00, 5.31it/s] -Transcribing: 100%|█████████▉| 2619/2620 [12:16<00:00, 5.41it/s] -Transcribing: 100%|██████████| 2620/2620 [12:16<00:00, 5.20it/s] -Transcribing: 100%|██████████| 2620/2620 [12:16<00:00, 3.56it/s] -Results saved at path: ./results/MODEL_openai-OpenVINO-whisper-large-v3-turbo-fp16-ov_DATASET_hf-audio-esb-datasets-test-only-sorted_librispeech_test.clean.jsonl -WER: 1.97 % -RTFx: 28.87 -``` +**Deploying on Bare Metal** -Where: -- WER (Word Error Rate) is the percentage of transcription errors compared to the reference text (substitutions + deletions + insertions). Lower is better. -- RTFx (Real-Time Factor, expressed as speedup) indicates processing speed relative to audio duration. Values above 1 mean faster-than-real-time transcription (for example, 5.16 means about 5.16x real time). +```bat +mkdir models -**For Open ASR Leaderboard, run `run_eval.py` with model name prefixed by `openai/` (for example `openai/whisper-large-v3-turbo-fp16-ov`).** -**OVMS should still be deployed with `--model_name whisper-large-v3-turbo-fp16-ov` (evaluation script does not include `openai/` prefix in requests).** -You can replace `librispeech` with other datasets supported by the leaderboard configuration. For multilingual models run_eval_ml.py should be used. +python export_model.py text2speech --source_model Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish --weight-format fp16 --model_name speecht5_tts_spanish --config_file_path models/config.json --model_repository_path models --overwrite_models --vocoder microsoft/speecht5_hifigan -## Translation -To test the translations endpoint we first need to prepare an audio file with speech in a language other than English, e.g. Spanish. To generate such a sample, follow the [Speech generation](#speech-generation) section to deploy Kokoro and then run: +ovms --rest_port 8000 --model_path models/speecht5_tts_spanish --model_name speecht5_tts_spanish -```console -curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"Kokoro-82M-OpenVINO-FP16-OVMS\", \"voice\": \"em_alex\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav +curl http://localhost:8000/v3/audio/speech -H "Content-Type: application/json" -d "{\"model\": \"speecht5_tts_spanish\", \"input\": \"Madrid es la capital de España\"}" -o speech_spanish.wav ``` -### Deployment +### Model preparation Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. Here is an example of OpenVINO/whisper-large-v3-fp16-ov deployment: @@ -389,7 +399,7 @@ Select deployment option depending on how you prepared models in the previous st Running this command starts the container with CPU only target device: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device CPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text ``` **GPU** @@ -398,7 +408,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device GPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU ``` ::: @@ -408,22 +418,22 @@ If you run on GPU make sure to have appropriate drivers installed, so the device ```bat mkdir models -ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device CPU +ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device CPU ``` or ```bat -ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name whisper-large-v3-fp16-ov --task speech2text --target_device GPU +ovms --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU ``` ::: ### Request Generation -Translate the speech_spanish.wav file generated above. +Transcript and translate file that was previously generated with audio/speech endpoint. :::{dropdown} **Unary call with cURL** ```bash -curl http://localhost:8000/v3/audio/translations -H "Content-Type: multipart/form-data" -F file="@speech_spanish.wav" -F model="whisper-large-v3-fp16-ov" +curl http://localhost:8000/v3/audio/translations -H "Content-Type: multipart/form-data" -F file="@speech_spanish.wav" -F model="OpenVINO/whisper-large-v3-fp16-ov" ``` ```json {"text": " Madrid is the capital of Spain."} @@ -445,7 +455,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") translation = client.audio.translations.create( - model="whisper-large-v3-fp16-ov", + model="OpenVINO/whisper-large-v3-fp16-ov", file=audio_file ) diff --git a/demos/benchmark/v3/benchmark.py b/demos/benchmark/v3/benchmark.py index b6fdeb3ad6..88c9aa8f18 100644 --- a/demos/benchmark/v3/benchmark.py +++ b/demos/benchmark/v3/benchmark.py @@ -52,7 +52,6 @@ parser.add_argument('--model', required=False, default='Alibaba-NLP/gte-large-en-v1.5', help='HF model name', dest='model') parser.add_argument('--request_rate', required=False, default='inf', help='Average amount of requests per seconds in random distribution', dest='request_rate') parser.add_argument('--batch_size', required=False, type=int, default=16, help='Number of strings in every requests', dest='batch_size') -parser.add_argument('--voice', required=False, help='Voice name for text2speech backend', dest='voice') parser.add_argument('--backend', required=False, default='ovms-embeddings', choices=['ovms-embeddings','tei-embed','infinity-embeddings','ovms_rerank','text2speech','speech2text', 'translations'], help='Backend serving API type', dest='backend') parser.add_argument('--limit', required=False, type=int, default=1000, help='Number of documents to use in testing', dest='limit') parser.add_argument('--split', required=False, default='train', help='Dataset split', dest='split') @@ -105,7 +104,6 @@ class RequestFuncOutput: success: bool = False latency: float = 0.0 tokens_len: int = 0 - audio_duration: float = 0.0 error: str = "" text: str = "" @@ -130,13 +128,10 @@ async def async_request_text2speech( "model": request_func_input.model, "input": request_func_input.documents[0], } - if args["voice"] is not None: - payload["voice"] = args["voice"] headers = application_json_headers output = RequestFuncOutput() st = time.perf_counter() - audio_bytes = bytearray() try: async with session.post(url=api_url, json=payload, headers=headers) as response: @@ -144,14 +139,13 @@ async def async_request_text2speech( async for chunk_bytes in response.content: if not chunk_bytes: continue - audio_bytes.extend(chunk_bytes) - output.success = True - output.latency = time.perf_counter() - st - try: - output.audio_duration = soundfile.info(io.BytesIO(audio_bytes)).duration - except Exception as e: - output.success = False - output.error = f"Could not decode WAV response to compute audio duration: {e}" + # uncomment for response debugging + # chunk_bytes = chunk_bytes.decode("utf-8") + # data = json.loads(chunk_bytes) + # TBD: saving response to file + timestamp = time.perf_counter() + output.success = True + output.latency = timestamp - st else: output.error = response.reason or "" output.success = False @@ -393,7 +387,6 @@ async def limited_request_func(request_func_input, pbar): "latencies": [output.latency for output in outputs], "successes": [output.success for output in outputs], "token_count": [output.tokens_len for output in outputs], - "audio_durations": [output.audio_duration for output in outputs], } return result @@ -435,47 +428,18 @@ async def limited_request_func(request_func_input, pbar): args["api_url"] = default_api_url benchmark_results = asyncio.run(benchmark(docs=docs, model=args["model"], api_url=args["api_url"], request_rate=float(args["request_rate"]), backend_function=backend_function)) - -success_count = sum(benchmark_results['successes']) -total_count = len(benchmark_results['successes']) -success_rate = (success_count / total_count * 100.0) if total_count else 0.0 - -if args["backend"] == "text2speech": - rtfx_values = [] - for latency, audio_duration, success in zip( - benchmark_results['latencies'], - benchmark_results['audio_durations'], - benchmark_results['successes'], - ): - if success and latency > 0 and audio_duration > 0: - # RTFx = generated audio seconds per one second of processing time. - rtfx_values.append(audio_duration / latency) - - print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") - print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") - print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") - - if len(rtfx_values) > 0: - print(f"Mean RTFx: {np.mean(rtfx_values):.3f}x") - print(f"Median RTFx: {np.median(rtfx_values):.3f}x") - else: - print("Mean RTFx: n/a") - print("Median RTFx: n/a") -elif args["backend"] == "speech2text" or args["backend"] == "translations": +if args["backend"] == "speech2text" or args["backend"] == "translations": num_tokens = sum(benchmark_results['token_count']) - #print(benchmark_results) - print("Tokens:",num_tokens) - print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") - print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") - print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") - print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") - print(f"Average document length: {num_tokens / len(docs)} tokens") else: num_tokens = count_tokens(docs=docs,model=args["model"]) - #print(benchmark_results) - print("Tokens:",num_tokens) - print(f"Success rate: {success_rate}%. ({success_count}/{total_count})") - print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") - print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") - print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") - print(f"Average document length: {num_tokens / len(docs)} tokens") +#print(benchmark_results) +print("Tokens:",num_tokens) +print(f"Success rate: {sum(benchmark_results['successes'])/len(benchmark_results['successes'])*100}%. ({sum(benchmark_results['successes'])}/{len(benchmark_results['successes'])})") +print(f"Throughput - Tokens per second: {num_tokens / benchmark_results['duration']:^,.1f}") +print(f"Mean latency: {np.mean(benchmark_results['latencies'])*1000:.2f} ms") +print(f"Median latency: {np.median(benchmark_results['latencies'])*1000:.2f} ms") +# add printing 10 percentiles of latency to better understand latency distribution +percentiles = [10, 25, 50, 75, 90, 95, 99] +for p in percentiles: + print(f"{p}th percentile latency: {np.percentile(benchmark_results['latencies'], p)*1000:.2f} ms") +print(f"Average document length: {num_tokens / len(docs)} tokens") From c54e9a69e32dc43126bff12f6a5c41604ef0b8b0 Mon Sep 17 00:00:00 2001 From: Michal Kulakowski Date: Fri, 17 Jul 2026 12:42:11 +0200 Subject: [PATCH 11/11] as above --- demos/audio/README.md | 201 +++++++++++++++++++++++++++++------------- 1 file changed, 139 insertions(+), 62 deletions(-) diff --git a/demos/audio/README.md b/demos/audio/README.md index 8b13a657b1..a59cd77c11 100644 --- a/demos/audio/README.md +++ b/demos/audio/README.md @@ -8,7 +8,7 @@ Check supported [Speech Recognition Models](https://openvinotoolkit.github.io/op ## Prerequisites -**OVMS version 2025.4** This demo require version 2025.4 or nightly release. +**OVMS version 2025.4** This demo requires version 2025.4 or nightly release. **Model preparation**: Python 3.10 or higher with pip @@ -155,7 +155,7 @@ An asynchronous benchmarking client can be used to access the model server perfo ```console pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py -python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model microsoft/speecht5_tts --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --tokenizer openai/whisper-large-v3-turbo --trust-remote-code True +python benchmark.py --api_url http://localhost:8000/v3/audio/speech --model microsoft/speecht5_tts --batch_size 1 --limit 100 --request_rate inf --backend text2speech --dataset edinburghcstr/ami --hf-subset ihm --tokenizer OpenVINO/whisper-large-v3-turbo-fp16-ov --trust-remote-code True Number of documents: 100 100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [01:58<00:00, 1.19s/it] Tokens: 1802 @@ -169,37 +169,7 @@ Average document length: 18.02 tokens ## Transcription ### Model preparation Many variances of Whisper models can be deployed in a single command by using pre-configured models from [OpenVINO HuggingFace organization](https://huggingface.co/collections/OpenVINO/speech-to-text) and used both for translations and transcriptions endpoints. -However in this demo we will use openai/whisper-large-v3-turbo which needs to be converted to IR format before using in OVMS. - -Here, the original Speech to Text model will be converted to IR format and quantized. -That ensures faster initialization time, better performance and lower memory consumption. -Execution parameters will be defined inside the `graph.pbtxt` file. - -Download export script, install it's dependencies and create directory for the models: -```console -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt -mkdir models -``` - -Run `export_model.py` script to download and quantize the model: - -> **Note:** The users in China need to set environment variable HF_ENDPOINT="https://hf-mirror.com" before running the export script to connect to the HF Hub. - -**CPU** -```console -python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name openai/whisper-large-v3-turbo --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps -``` - -**GPU** -```console -python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name openai/whisper-large-v3-turbo --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps --target_device GPU -``` - -> **Note:** Change the `--weight-format` to quantize the model to `int8` precision to reduce memory consumption and improve performance. -> **Note:** `--enable_word_timestamps` can be omitted if there is no need for word timestamps support. - -### Deployment +In this demo we will use OpenVINO/whisper-large-v3-turbo-fp16-ov, which is a fine-tuned version of Whisper large-v3. :::{dropdown} **Deploying with Docker** @@ -210,7 +180,7 @@ Select deployment option depending on how you prepared models in the previous st Running this command starts the container with CPU only target device: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path /models ``` **GPU** @@ -219,7 +189,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path /models --target_device GPU ``` ::: @@ -228,11 +198,13 @@ docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-a If you run on GPU make sure to have appropriate drivers installed, so the device is accessible for the model server. ```bat -ovms --rest_port 8000 --model_path /models/openai/whisper-large-v3-turbo --model_name openai/whisper-large-v3-turbo +ovms --rest_port 8000 --task speech2text --source_model OpenVINO/whisper-large-v3-turbo-fp16-ov --model_name OpenVINO/whisper-large-v3-turbo-fp16-ov --model_repository_path models --target_device GPU ``` ::: -The default configuration should work in most cases but the parameters can be tuned via `export_model.py` script arguments. Run the script with `--help` argument to check available parameters and see the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. +> **Note:** Sentence timestamps are supported via `timestamp_granularities[]=segment` in transcription requests. + +The default configuration should work in most cases but the parameters can be tuned via OVMS arguments. See the [s2t calculator documentation](../../docs/speech_recognition/reference.md) to learn more about configuration options and limitations. ### Request Generation Transcript file that was previously generated with audio/speech endpoint. @@ -243,7 +215,7 @@ Transcript file that was previously generated with audio/speech endpoint. ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="openai/whisper-large-v3-turbo" -F language="en" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" -F language="en" ``` ```json {"text": " The quick brown fox jumped over the lazy dog."} @@ -265,7 +237,7 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="openai/whisper-large-v3-turbo", + model="OpenVINO/whisper-large-v3-turbo-fp16-ov", language="en", file=audio_file ) @@ -283,7 +255,7 @@ The quick brown fox jumped over the lazy dog. curl -N http://localhost:8000/v3/audio/transcriptions \ -H "Content-Type: multipart/form-data" \ -F file="@speech.wav" \ - -F model="openai/whisper-large-v3-turbo" \ + -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" \ -F language="en" \ -F stream="true" ``` @@ -298,18 +270,18 @@ data: {"type":"transcript.text.done","text":"The quick brown fox jumped over the ``` ::: -:::{dropdown} **Unary call with timestamps** +:::{dropdown} **Unary call with sentence timestamps** ```bash -curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="openai/whisper-large-v3-turbo" -F language="en" -F timestamp_granularities[]="segment" -F timestamp_granularities[]="word" +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="OpenVINO/whisper-large-v3-turbo-fp16-ov" -F language="en" -F timestamp_granularities[]="segment" ``` ```json -{"text":" A quick brown fox jumped over the lazy dog","words":[{"word":" A","start":0.0,"end":0.14000000059604645},{"word":" quick","start":0.14000000059604645,"end":0.3400000035762787},{"word":" brown","start":0.3400000035762787,"end":0.7799999713897705},{"word":" fox","start":0.7799999713897705,"end":1.3199999332427979},{"word":" jumped","start":1.3199999332427979,"end":1.7799999713897705},{"word":" over","start":1.7799999713897705,"end":2.0799999237060547},{"word":" the","start":2.0799999237060547,"end":2.259999990463257},{"word":" lazy","start":2.259999990463257,"end":2.5399999618530273},{"word":" dog","start":2.5399999618530273,"end":2.919999837875366}],"segments":[{"text":" A quick brown fox jumped over the lazy dog","start":0.0,"end":3.1399998664855957}]} +{"text":" A quick brown fox jumped over the lazy dog","segments":[{"text":" A quick brown fox jumped over the lazy dog","start":0.0,"end":3.1399998664855957}]} ``` ::: -:::{dropdown} **Unary call with python OpenAI library with timestamps** +:::{dropdown} **Unary call with python OpenAI library with sentence timestamps** ```python from pathlib import Path @@ -324,40 +296,145 @@ client = OpenAI(base_url=url, api_key="not_used") audio_file = open(filename, "rb") transcript = client.audio.transcriptions.create( - model="openai/whisper-large-v3-turbo", + model="OpenVINO/whisper-large-v3-turbo-fp16-ov", language="en", response_format="verbose_json", - timestamp_granularities=["segment", "word"], + timestamp_granularities=["segment"], file=audio_file ) print(transcript.text) print(transcript.segments) -print(transcript.words) ``` ``` A quick brown fox jumped over the lazy dog [TranscriptionSegment(id=None, avg_logprob=None, compression_ratio=None, end=3.1399998664855957, no_speech_prob=None, seek=None, start=0.0, temperature=None, text=' A quick brown fox jumped over the lazy dog', tokens=None)] -[TranscriptionWord(end=0.14000000059604645, start=0.0, word=' A'), TranscriptionWord(end=0.3400000035762787, start=0.14000000059604645, word=' quick'), TranscriptionWord(end=0.7799999713897705, start=0.3400000035762787, word=' brown'), TranscriptionWord(end=1.3199999332427979, start=0.7799999713897705, word=' fox'), TranscriptionWord(end=1.7799999713897705, start=1.3199999332427979, word=' jumped'), TranscriptionWord(end=2.0799999237060547, start=1.7799999713897705, word=' over'), TranscriptionWord(end=2.259999990463257, start=2.0799999237060547, word=' the'), TranscriptionWord(end=2.5399999618530273, start=2.259999990463257, word=' lazy'), TranscriptionWord(end=2.919999837875366, start=2.5399999618530273, word=' dog')] ``` ::: -## Benchmarking transcription -An asynchronous benchmarking client can be used to access the model server performance with various load conditions. Below are execution examples captured on Intel(R) Core(TM) Ultra 7 258V. +### Word timestamps +If you need word-level timestamps support, export the model with `export_model.py` and enable this feature during export. +Prepare export script and dependencies: ```console -pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/requirements.txt -curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/benchmark/v3/benchmark.py -o benchmark.py -python benchmark.py --api_url http://localhost:8000/v3/audio/transcriptions --model openai/whisper-large-v3-turbo --batch_size 1 --limit 1000 --request_rate inf --dataset edinburghcstr/ami --hf-subset ihm --backend speech2text --trust-remote-code True -Number of documents: 1000 -100%|██████████████████████████████████████████████████████████████████████████████| 1000/1000 [04:44<00:00, 3.51it/s] -Tokens: 10948 -Success rate: 100.0%. (1000/1000) -Throughput - Tokens per second: 38.5 -Mean latency: 26670.64 ms -Median latency: 20772.09 ms -Average document length: 10.948 tokens +curl https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/export_model.py -o export_model.py +pip install -r https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/common/export_models/requirements.txt +mkdir -p models +``` + +Export Speech-to-Text model with word timestamps enabled: +```console +python export_model.py speech2text --source_model openai/whisper-large-v3-turbo --weight-format fp16 --model_name whisper-large-v3-turbo-word-ts --config_file_path models/config.json --model_repository_path models --overwrite_models --enable_word_timestamps +``` + +:::{dropdown} **Deploying with Docker** + +```bash +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 -v $(pwd)/models:/models:rw openvino/model_server:latest --rest_port 8000 --model_path /models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts +``` +::: + +:::{dropdown} **Deploying on Bare Metal** + +```bat +ovms --rest_port 8000 --model_path models/whisper-large-v3-turbo-word-ts --model_name whisper-large-v3-turbo-word-ts +``` +::: + +:::{dropdown} **Unary call with cURL (word timestamps)** + +```bash +curl http://localhost:8000/v3/audio/transcriptions -H "Content-Type: multipart/form-data" -F file="@speech.wav" -F model="whisper-large-v3-turbo-word-ts" -F language="en" -F timestamp_granularities[]="word" +``` + +Example response: +```json +{"text":" A quick brown fox jumped over the lazy dog","words":[{"word":" A","start":0.0,"end":0.14000000059604645},{"word":" quick","start":0.14000000059604645,"end":0.3400000035762787},{"word":" brown","start":0.3400000035762787,"end":0.7799999713897705},{"word":" fox","start":0.7799999713897705,"end":1.3199999332427979},{"word":" jumped","start":1.3199999332427979,"end":1.7799999713897705},{"word":" over","start":1.7799999713897705,"end":2.0799999237060547},{"word":" the","start":2.0799999237060547,"end":2.259999990463257},{"word":" lazy","start":2.259999990463257,"end":2.5399999618530273},{"word":" dog","start":2.5399999618530273,"end":2.919999837875366}]} +``` +::: + +:::{dropdown} **Unary call with python OpenAI library (word timestamps)** + +```python +from pathlib import Path +from openai import OpenAI + +filename = "speech.wav" +url="http://localhost:8000/v3" + + +speech_file_path = Path(__file__).parent / filename +client = OpenAI(base_url=url, api_key="not_used") + +audio_file = open(filename, "rb") +transcript = client.audio.transcriptions.create( + model="whisper-large-v3-turbo-word-ts", + language="en", + response_format="verbose_json", + timestamp_granularities=["word"], + file=audio_file +) + +print(transcript.text) +print(transcript.words) ``` +::: + +## Evaluate transcription accuracy and performance with Open ASR Leaderboard + +You can evaluate model accuracy (for example WER/CER) against ASR datasets using the Open ASR Leaderboard tooling. + +Clone the repository: +```console +git clone https://github.com/huggingface/open_asr_leaderboard.git +cd open_asr_leaderboard +``` + +Download and apply OVMS API compatibility patch: + + curl -L https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/external/open_asr_leaderboard.patch -o ovms_open_asr_leaderboard.patch + git apply ovms_open_asr_leaderboard.patch + +Set OpenAI-compatible endpoint variables for OVMS: +```console +export OPENAI_BASE_URL=http://localhost:8000/v3 +export OPENAI_API_KEY="unused" +``` + +Install dependencies: +```console +pip install -r requirements/requirements.txt -r requirements/requirements-api.txt openai>=1.0.0 torchcodec==0.12 +``` + +Run evaluation example: +```console +PYTHONPATH=. python api/run_eval.py \ + --model_name openai/OpenVINO/whisper-large-v3-turbo-fp16-ov \ + --dataset_path "hf-audio/esb-datasets-test-only-sorted" \ + --max_workers 1 \ + --split test.clean \ + --dataset "librispeech" +``` +Results: +```console +... +Transcribing: 100%|█████████▉| 2617/2620 [12:15<00:00, 5.23it/s] +Transcribing: 100%|█████████▉| 2618/2620 [12:15<00:00, 5.31it/s] +Transcribing: 100%|█████████▉| 2619/2620 [12:16<00:00, 5.41it/s] +Transcribing: 100%|██████████| 2620/2620 [12:16<00:00, 5.20it/s] +Transcribing: 100%|██████████| 2620/2620 [12:16<00:00, 3.56it/s] +Results saved at path: ./results/MODEL_openai-OpenVINO-whisper-large-v3-turbo-fp16-ov_DATASET_hf-audio-esb-datasets-test-only-sorted_librispeech_test.clean.jsonl +WER: 1.97 % +RTFx: 28.87 +``` + +Where: +- WER (Word Error Rate) is the percentage of transcription errors compared to the reference text (substitutions + deletions + insertions). Lower is better. +- RTFx (Real-Time Factor, expressed as speedup) indicates processing speed relative to audio duration. Values above 1 mean faster-than-real-time transcription (for example, 5.16 means about 5.16x real time). + +**For Open ASR Leaderboard, run `run_eval.py` with model name prefixed by `openai/` (for example `openai/OpenVINO/whisper-large-v3-turbo-fp16-ov`).** +**OVMS should still be deployed with `--model_name OpenVINO/whisper-large-v3-turbo-fp16-ov` (evaluation script does not include `openai/` prefix in requests).** +You can replace `librispeech` with other datasets supported by the leaderboard configuration. For multilingual models run_eval_ml.py should be used. ## Translation To test translations endpoint we first need to prepare audio file with speech in language other than English, e.g. Spanish. To generate such sample we will use finetuned version of microsoft/speecht5_tts model. @@ -408,7 +485,7 @@ to `docker run` command, use the image with GPU support. It can be applied using the commands below: ```bash mkdir -p models -docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU +docker run -d -u $(id -u):$(id -g) --rm -p 8000:8000 --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -v $(pwd)/models:/models:rw openvino/model_server:latest-gpu --rest_port 8000 --source_model OpenVINO/whisper-large-v3-fp16-ov --model_repository_path /models --model_name OpenVINO/whisper-large-v3-fp16-ov --task speech2text --target_device GPU ``` :::