CVS-188170-remove-tfs

tests/functional/constants/metrics.py

@@ -69,13 +69,9 @@ class Metric:
 
     Method_infer = "ModelInfer"
 
-    Method_getmodelstatus = "GetModelStatus"
-    Method_getmodelmetadata = "GetModelMetadata"
-    Method_predict = "Predict"
     Method_modelready = "ModelReady"
     Method_modelmetadata = "ModelMetadata"
 
-    TensorFlowServing = "TensorFlowServing"
     KServe = "KServe"
 
     Type_counter = "counter"
@@ -118,12 +114,9 @@ class Metric:
 
     DescType = {"requests_success", "counter", "request_fail", "counter"}
 
-    Protocol = [KServe, TensorFlowServing]
+    Protocol = [KServe]
 
     Methods = [
-        Method_getmodelstatus,
-        Method_getmodelmetadata,
-        Method_predict,
         Method_infer,
         Method_modelready,
         Method_modelmetadata,
@@ -133,9 +126,6 @@ class Metric:
         Method_modelready: KServe,
         Method_modelmetadata: KServe,
         Method_infer: KServe,
-        Method_getmodelstatus: TensorFlowServing,
-        Method_getmodelmetadata: TensorFlowServing,
-        Method_predict: TensorFlowServing,
     }
 
     Histogram_bucket_len_list = [
@@ -194,7 +184,7 @@ def create_method_metrics(model, base_name):
                     "name": model.name,
                 }
 
-                if method not in [Metric.Method_getmodelstatus, Metric.Method_modelready]:
+                if method not in [Metric.Method_modelready]:
                     content["version"] = str(model.version)
 
                 result.append(Metric(metric_name=base_name, content=content))
@@ -384,19 +374,15 @@ def create_from_model_list(model_list, ovms_run=None, metrics=None):
 
         """ 
         The following metrics are not multiplied for each model version (should occur once for single model name)
-        ovms_requests_success[{'api': 'TensorFlowServing', 'interface': 'gRPC', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0
-        ovms_requests_success[{'api': 'TensorFlowServing', 'interface': 'REST', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0
         ovms_requests_success[{'api': 'KServe', 'interface': 'gRPC', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0
         ovms_requests_success[{'api': 'KServe', 'interface': 'REST', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0
-        ovms_requests_fail[{'api': 'TensorFlowServing', 'interface': 'gRPC', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0
-        ovms_requests_fail[{'api': 'TensorFlowServing', 'interface': 'REST', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0
         ovms_requests_fail[{'api': 'KServe', 'interface': 'gRPC', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0
         ovms_requests_fail[{'api': 'KServe', 'interface': 'REST', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0
         """
         metrics_to_remove = []
         model_unique_metrics = []
         for metric in metric_list:
-            if metric.content.get("method", None) in [Metric.Method_getmodelstatus, Metric.Method_modelready]:
+            if metric.content.get("method", None) in [Metric.Method_modelready]:
                 if metric.to_str() in model_unique_metrics:
                     metrics_to_remove.append(metric)
                 else:
@@ -541,40 +527,12 @@ def __init__(self):
 #    protocol="kserve",
 #    version="1"} 0
 # ovms_requests_success{
-#    interface="rest",
-#    method="getmodelstatus",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving"} 0
-# ovms_requests_success{
-#    interface="rest",
-#    method="getmodelmetadata",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_success{
-#    interface="rest",
-#    method="predict",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_success{
 #    interface="grpc",
 #    method="modelinfer",
 #    name="ssdlite_mobilenet_v2_ov",
 #    protocol="kserve",
 #    version="1"} 0
 # ovms_requests_success{
-#    interface="grpc",
-#    method="getmodelstatus",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving"} 0
-# ovms_requests_success{
-#    interface="grpc",
-#    method="getmodelmetadata",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_success{
 #    interface="rest",
 #    method="modelready",
 #    name="ssdlite_mobilenet_v2_ov",
@@ -585,12 +543,6 @@ def __init__(self):
 #    name="ssdlite_mobilenet_v2_ov",
 #    protocol="kserve",
 #    version="1"} 0
-# ovms_requests_success{
-#    interface="grpc",
-#    method="predict",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
 # # HELP ovms_requests_fail Number of failed requests to a model or a DAG.
 # # TYPE ovms_requests_fail counter
 # ovms_requests_fail{
@@ -618,53 +570,17 @@ def __init__(self):
 #    protocol="kserve",
 #    version="1"} 0
 # ovms_requests_fail{
-#    interface="rest",
-#    method="getmodelstatus",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_fail{
-#    interface="rest",
-#    method="getmodelmetadata",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_fail{
-#    interface="rest",
-#    method="predict",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_fail{
 #    interface="grpc",
 #    method="modelmetadata",
 #    name="ssdlite_mobilenet_v2_ov",
 #    protocol="kserve",
 #    version="1"} 0
 # ovms_requests_fail{
-#    interface="grpc",
-#    method="getmodelstatus",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_fail{
-#    interface="grpc",
-#    method="getmodelmetadata",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
-# ovms_requests_fail{
 #    interface="rest",
 #    method="modelmetadata",
 #    name="ssdlite_mobilenet_v2_ov",
 #    protocol="kserve",
 #    version="1"} 0
-# ovms_requests_fail{
-#    interface="grpc",
-#    method="predict",
-#    name="ssdlite_mobilenet_v2_ov",
-#    protocol="tensorflowserving",
-#    version="1"} 0
 # # HELP ovms_streams Number of OpenVINO execution streams.
 # # TYPE ovms_streams gauge
 # ovms_streams{name="ssdlite_mobilenet_v2_ov",version="1"} 4

tests/functional/constants/ovms.py

@@ -18,7 +18,6 @@
 import re
 
 from enum import Enum
-from tensorflow_serving.apis.get_model_status_pb2 import ModelVersionStatus
 
 from tests.functional.constants.os_type import OsType
 
@@ -137,12 +136,12 @@ class Ovms:
 
     class ModelStatus(Enum):
         UNDEFINED = None
-        UNKNOWN = ModelVersionStatus.UNKNOWN
-        START = ModelVersionStatus.START
-        LOADING = ModelVersionStatus.LOADING
-        AVAILABLE = ModelVersionStatus.AVAILABLE
-        UNLOADING = ModelVersionStatus.UNLOADING
-        END = ModelVersionStatus.END
+        UNKNOWN = 0
+        START = 10
+        LOADING = 20
+        AVAILABLE = 30
+        UNLOADING = 40
+        END = 50
 
     LAYOUT_NHWC = "NHWC:NCHW"
     LAYOUT_NCHW = "NCHW:NCHW"

tests/functional/fixtures/api_type.py

@@ -21,7 +21,7 @@
 from tests.functional.constants.ovms_type import OvmsType
 from tests.functional.utils.inference.communication import GRPC, REST
 from tests.functional.utils.inference.inference_client_factory import InferenceClientFactory
-from tests.functional.utils.inference.serving import KFS, OPENAI, TFS, TRITON, COHERE
+from tests.functional.utils.inference.serving import KFS, OPENAI, TRITON, COHERE
 
 
 def api_type_non_fixture(serving, communication, ovms_type=None):
@@ -41,18 +41,13 @@ def api_type(request):
         return api_type_non_fixture(*request.param, ovms_type=None)
 
 
-@pytest.fixture(scope="session", params=itertools.product([TFS], [GRPC, REST]), ids=lambda x: f":".join(x).upper())
-def tfs_api_type(request):
-    return api_type_non_fixture(*request.param)
-
-
-@pytest.fixture(scope="session", params=[(TFS, REST)], ids=lambda x: f":".join(x).upper())
-def tfs_rest_api_type(request):
+@pytest.fixture(scope="session", params=itertools.product([KFS], [REST]), ids=lambda x: f":".join(x).upper())
+def rest_api_type(request):
     return api_type_non_fixture(*request.param)
 
 
-@pytest.fixture(scope="session", params=[(TFS, GRPC)], ids=lambda x: f":".join(x).upper())
-def tfs_grpc_api_type(request):
+@pytest.fixture(scope="session", params=itertools.product([KFS], [GRPC]), ids=lambda x: f":".join(x).upper())
+def grpc_api_type(request):
     return api_type_non_fixture(*request.param)
 
 
@@ -94,13 +89,3 @@ def triton_grpc_api_type(request):
 @pytest.fixture(scope="session", params=[(TRITON, REST)], ids=lambda x: f":".join(x).upper())
 def triton_rest_api_type(request):
     return api_type_non_fixture(*request.param)
-
-
-@pytest.fixture(scope="session", params=itertools.product([KFS, TFS], [REST]), ids=lambda x: f":".join(x).upper())
-def rest_api_type(request):
-    return api_type_non_fixture(*request.param)
-
-
-@pytest.fixture(scope="session", params=itertools.product([KFS, TFS], [GRPC]), ids=lambda x: f":".join(x).upper())
-def grpc_api_type(request):
-    return api_type_non_fixture(*request.param)

tests/functional/object_model/inference_helpers.py

@@ -40,9 +40,6 @@
 from openai import OpenAI
 from pydantic import BaseModel
 from retry.api import retry_call
-from tensorflow import make_tensor_proto
-from tensorflow_serving.apis import get_model_status_pb2
-from tensorflow_serving.apis.predict_pb2 import PredictRequest
 from tritonclient.grpc import service_pb2, service_pb2_grpc
 from tritonclient.grpc.service_pb2 import ModelInferRequest
 from tritonclient.utils import InferenceServerException, deserialize_bytes_tensor, serialize_byte_tensor
@@ -62,7 +59,6 @@
     AudioApi,
     ResponsesApi,
 )
-from tests.functional.utils.inference.serving.tf import TensorFlowServingWrapper
 from tests.functional.utils.logger import get_logger
 from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime
 from tests.functional.utils.generative_ai.validation_utils import GenerativeAIValidationUtils
@@ -223,14 +219,6 @@ def prepare_request_to_send(self, client, input_data):
             request = {"request": request}
         elif isinstance(client, KserveWrapper) and isinstance(client, RestCommunicationInterface):
             request = self._create_kfs_post_request(input_data)
-        elif isinstance(client, TensorFlowServingWrapper) and isinstance(client, RestCommunicationInterface):
-            request = self._create_post_request(self.model.input_names, input_data, request_format=self.layout)
-        elif isinstance(client, TensorFlowServingWrapper) and isinstance(client, GrpcCommunicationInterface):
-            request = PredictRequest()
-            request.model_spec.name = self.model.name
-            for input_name, input_object in input_data.items():
-                request.inputs[input_name].CopyFrom(make_tensor_proto(input_object, shape=[len(input_object)]))
-            request = {"request": request}
         else:
             raise NotImplementedError
         return request
@@ -824,78 +812,22 @@ def prepare_and_run_set_of_predict_requests(ovms: OvmsInstance, models, api_type
 def get_model_status(client, accepted_model_states=None, model_version=None, port=None):
     model_state = None
     port = port if port is not None else client.port
-    if client.serving == KFS:
-        if accepted_model_states is not None:
-            for elem in accepted_model_states:
-                is_ready = True if elem == Ovms.ModelStatus.AVAILABLE else False
-                try:
-                    model_state = check_model_readiness(client.model, port, type(client), timeout=30, is_ready=is_ready)
-                except ModelNotReadyException:
-                    logger.info(f"Model state not in accepted state: {elem}")
-                finally:
-                    break
-            else:
-                raise ModelNotReadyException(f"Failed to check model: {client.model}")
+    if accepted_model_states is not None:
+        for elem in accepted_model_states:
+            is_ready = True if elem == Ovms.ModelStatus.AVAILABLE else False
+            try:
+                model_state = check_model_readiness(client.model, port, type(client), timeout=30, is_ready=is_ready)
+            except ModelNotReadyException:
+                logger.info(f"Model state not in accepted state: {elem}")
+            finally:
+                break
         else:
-            model_state = check_model_readiness(client.model, port, type(client))
+            raise ModelNotReadyException(f"Failed to check model: {client.model}")
     else:
-        status = client.get_model_status()
-        logger.debug(f"status: {status}")
-        if model_version is None:
-            model_state = Ovms.ModelStatus(status.model_version_status[0].state)
-        else:
-            for model_version_status in status.model_version_status:
-                if model_version_status.version == model_version:
-                    model_state = Ovms.ModelStatus(model_version_status.state)
-                    break
-        if accepted_model_states:
-            if model_state not in accepted_model_states:
-                model_str_name = client.model_name
-                raise ValueError(f"Incorrect state of {model_str_name}: {model_state}")
+        model_state = check_model_readiness(client.model, port, type(client))
     return model_state
 
 
-def get_and_validate_model_status(inference, expected_models_status):
-    status = inference.get_model_status()
-    if expected_models_status is not None:
-        assert len(expected_models_status) == len(status.model_version_status)
-
-    for i, model_version_status in enumerate(status.model_version_status):
-        model_state = model_version_status.state
-        error_message = model_version_status.status.error_message
-        version = model_version_status.version
-
-        if expected_models_status is None or expected_models_status[i].get("accepted_states", None) is None:
-            model_accepted_states = [
-                get_model_status_pb2.ModelVersionStatus.START,
-                get_model_status_pb2.ModelVersionStatus.AVAILABLE,
-                get_model_status_pb2.ModelVersionStatus.UNLOADING,
-                get_model_status_pb2.ModelVersionStatus.LOADING,
-                get_model_status_pb2.ModelVersionStatus.END,
-            ]
-        else:
-            model_accepted_states = expected_models_status[i]["accepted_states"]
-
-        if model_state not in model_accepted_states:
-            raise ValueError(f"Incorrect model state: {model_state}")
-
-        if expected_models_status is None or expected_models_status[i].get("accepted_error_messages", None) is None:
-            if model_state == get_model_status_pb2.ModelVersionStatus.LOADING:
-                model_accepted_error_messages = ["OK", "UNKNOWN"]
-            else:
-                model_accepted_error_messages = ["OK"]
-        else:
-            model_accepted_error_messages = expected_models_status[i]["accepted_error_messages"]
-
-        if error_message not in model_accepted_error_messages:
-            raise ValueError(f"Incorrect error message: {model_state}")
-
-        if expected_models_status is not None:
-            assert version == expected_models_status[i]["version"]
-
-    return status
-
-
 def get_multiple_model_status(models_and_expected_state):
     for client, state in models_and_expected_state:
         try: