chore(eval): move tabby-python-client to eval dir

Makefile

@@ -30,15 +30,3 @@ update-openapi-doc:
 			["components", "schemas", "DebugOptions"]                                                                 \
 			])' | jq '.servers[0] |= { url: "https://playground.app.tabbyml.com", description: "Playground server" }' \
 			    > website/static/openapi.json
-
-update-python-client:
-	rm -rf clients/tabby-python-client
-	curl http://localhost:8080/api-docs/openapi.json | jq '                                                       \
-	  delpaths([                                                                                                  \
-		  ["paths", "/v1beta/chat/completions"]                                                                     \
-		])' > /tmp/openapi.json
-
-	cd clients && openapi-python-client generate \
-    --path /tmp/openapi.json \
-    --config ../experimental/openapi/python.yaml \
-    --meta setup

clients/tabby-python-client/tabby_client/models/debug_options.py

@@ -1,57 +0,0 @@
-from typing import Any, Dict, List, Type, TypeVar
-
-import attr
-
-T = TypeVar("T", bound="DebugOptions")
-
-
-@attr.s(auto_attribs=True)
-class DebugOptions:
-    """
-    Attributes:
-        enabled (bool): When true, returns debug_data in completion response.
-    """
-
-    enabled: bool
-    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)
-
-    def to_dict(self) -> Dict[str, Any]:
-        enabled = self.enabled
-
-        field_dict: Dict[str, Any] = {}
-        field_dict.update(self.additional_properties)
-        field_dict.update(
-            {
-                "enabled": enabled,
-            }
-        )
-
-        return field_dict
-
-    @classmethod
-    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
-        d = src_dict.copy()
-        enabled = d.pop("enabled")
-
-        debug_options = cls(
-            enabled=enabled,
-        )
-
-        debug_options.additional_properties = d
-        return debug_options
-
-    @property
-    def additional_keys(self) -> List[str]:
-        return list(self.additional_properties.keys())
-
-    def __getitem__(self, key: str) -> Any:
-        return self.additional_properties[key]
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        self.additional_properties[key] = value
-
-    def __delitem__(self, key: str) -> None:
-        del self.additional_properties[key]
-
-    def __contains__(self, key: str) -> bool:
-        return key in self.additional_properties

experimental/eval/gen-client.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+rm -rf tabby-python-client
+
+curl http://localhost:8080/api-docs/openapi.json | jq 'delpaths([
+  ["paths", "/v1beta/chat/completions"]
+])' > /tmp/openapi.json
+
+openapi-python-client generate \
+  --path /tmp/openapi.json \
+  --config ./python.yaml \
+  --meta setup

experimental/eval/python.yaml

@@ -0,0 +1,2 @@
+project_name_override: tabby-python-client
+package_name_override: tabby_python_client

experimental/eval/tabby-python-client/README.md

@@ -5,7 +5,7 @@ A client library for accessing Tabby Server
 First, create a client:
 
 ```python
-from tabby_client import Client
+from tabby_python_client import Client
 
 client = Client(base_url="https://api.example.com")
 ```
@@ -13,7 +13,7 @@ client = Client(base_url="https://api.example.com")
 If the endpoints you're going to hit require authentication, use `AuthenticatedClient` instead:
 
 ```python
-from tabby_client import AuthenticatedClient
+from tabby_python_client import AuthenticatedClient
 
 client = AuthenticatedClient(base_url="https://api.example.com", token="SuperSecretToken")
 ```
@@ -21,9 +21,9 @@ client = AuthenticatedClient(base_url="https://api.example.com", token="SuperSec
 Now call your endpoint and use your models:
 
 ```python
-from tabby_client.models import MyDataModel
-from tabby_client.api.my_tag import get_my_data_model
-from tabby_client.types import Response
+from tabby_python_client.models import MyDataModel
+from tabby_python_client.api.my_tag import get_my_data_model
+from tabby_python_client.types import Response
 
 my_data: MyDataModel = get_my_data_model.sync(client=client)
 # or if you need more info (e.g. status_code)
@@ -33,9 +33,9 @@ response: Response[MyDataModel] = get_my_data_model.sync_detailed(client=client)
 Or do the same thing with an async version:
 
 ```python
-from tabby_client.models import MyDataModel
-from tabby_client.api.my_tag import get_my_data_model
-from tabby_client.types import Response
+from tabby_python_client.models import MyDataModel
+from tabby_python_client.api.my_tag import get_my_data_model
+from tabby_python_client.types import Response
 
 my_data: MyDataModel = await get_my_data_model.asyncio(client=client)
 response: Response[MyDataModel] = await get_my_data_model.asyncio_detailed(client=client)
@@ -72,7 +72,7 @@ Things to know:
 
 1. All path/query params, and bodies become method arguments.
 1. If your endpoint had any tags on it, the first tag will be used as a module name for the function (my_tag above)
-1. Any endpoint which did not have a tag will be in `tabby_client.api.default`
+1. Any endpoint which did not have a tag will be in `tabby_python_client.api.default`
 
 ## Building / publishing this Client
 This project uses [Poetry](https://python-poetry.org/) to manage dependencies  and packaging.  Here are the basics:

experimental/eval/tabby-python-client/setup.py

@@ -7,12 +7,12 @@ long_description = (here / "README.md").read_text(encoding="utf-8")
 
 setup(
     name="tabby-python-client",
-    version="0.3.1",
+    version="0.4.0-dev",
     description="A client library for accessing Tabby Server",
     long_description=long_description,
     long_description_content_type="text/markdown",
     packages=find_packages(),
     python_requires=">=3.8, <4",
     install_requires=["httpx >= 0.15.0, < 0.25.0", "attrs >= 21.3.0", "python-dateutil >= 2.8.0, < 3"],
-    package_data={"tabby_client": ["py.typed"]},
+    package_data={"tabby_python_client": ["py.typed"]},
 )

experimental/eval/tabby-python-client/tabby_python_client/models/completion_request.py

@@ -20,7 +20,6 @@ class CompletionRequest:
             fib(n - 2)'}}
 
     Attributes:
-        prompt (Union[Unset, None, str]):  Example: def fib(n):.
         language (Union[Unset, None, str]): Language identifier, full list is maintained at
             https://code.visualstudio.com/docs/languages/identifiers Example: python.
         segments (Union[Unset, None, Segments]):
@@ -30,7 +29,6 @@ class CompletionRequest:
         debug_options (Union[Unset, None, DebugOptions]):
     """
 
-    prompt: Union[Unset, None, str] = UNSET
     language: Union[Unset, None, str] = UNSET
     segments: Union[Unset, None, "Segments"] = UNSET
     user: Union[Unset, None, str] = UNSET
@@ -38,7 +36,6 @@ class CompletionRequest:
     additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
-        prompt = self.prompt
         language = self.language
         segments: Union[Unset, None, Dict[str, Any]] = UNSET
         if not isinstance(self.segments, Unset):
@@ -52,8 +49,6 @@ class CompletionRequest:
         field_dict: Dict[str, Any] = {}
         field_dict.update(self.additional_properties)
         field_dict.update({})
-        if prompt is not UNSET:
-            field_dict["prompt"] = prompt
         if language is not UNSET:
             field_dict["language"] = language
         if segments is not UNSET:
@@ -71,8 +66,6 @@ class CompletionRequest:
         from ..models.segments import Segments
 
         d = src_dict.copy()
-        prompt = d.pop("prompt", UNSET)
-
         language = d.pop("language", UNSET)
 
         _segments = d.pop("segments", UNSET)
@@ -96,7 +89,6 @@ class CompletionRequest:
             debug_options = DebugOptions.from_dict(_debug_options)
 
         completion_request = cls(
-            prompt=prompt,
             language=language,
             segments=segments,
             user=user,

experimental/eval/tabby-python-client/tabby_python_client/models/debug_data.py

@@ -15,33 +15,35 @@ T = TypeVar("T", bound="DebugData")
 class DebugData:
     """
     Attributes:
-        prompt (str):
-        snippets (Union[Unset, List['Snippet']]):
+        snippets (Union[Unset, None, List['Snippet']]):
+        prompt (Union[Unset, None, str]):
     """
 
-    prompt: str
-    snippets: Union[Unset, List["Snippet"]] = UNSET
+    snippets: Union[Unset, None, List["Snippet"]] = UNSET
+    prompt: Union[Unset, None, str] = UNSET
     additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
-        prompt = self.prompt
-        snippets: Union[Unset, List[Dict[str, Any]]] = UNSET
+        snippets: Union[Unset, None, List[Dict[str, Any]]] = UNSET
         if not isinstance(self.snippets, Unset):
-            snippets = []
-            for snippets_item_data in self.snippets:
-                snippets_item = snippets_item_data.to_dict()
+            if self.snippets is None:
+                snippets = None
+            else:
+                snippets = []
+                for snippets_item_data in self.snippets:
+                    snippets_item = snippets_item_data.to_dict()
 
-                snippets.append(snippets_item)
+                    snippets.append(snippets_item)
+
+        prompt = self.prompt
 
         field_dict: Dict[str, Any] = {}
         field_dict.update(self.additional_properties)
-        field_dict.update(
-            {
-                "prompt": prompt,
-            }
-        )
+        field_dict.update({})
         if snippets is not UNSET:
             field_dict["snippets"] = snippets
+        if prompt is not UNSET:
+            field_dict["prompt"] = prompt
 
         return field_dict
 
@@ -50,8 +52,6 @@ class DebugData:
         from ..models.snippet import Snippet
 
         d = src_dict.copy()
-        prompt = d.pop("prompt")
-
         snippets = []
         _snippets = d.pop("snippets", UNSET)
         for snippets_item_data in _snippets or []:
@@ -59,9 +59,11 @@ class DebugData:
 
             snippets.append(snippets_item)
 
+        prompt = d.pop("prompt", UNSET)
+
         debug_data = cls(
-            prompt=prompt,
             snippets=snippets,
+            prompt=prompt,
         )
 
         debug_data.additional_properties = d

experimental/eval/tabby-python-client/tabby_python_client/models/debug_options.py

@@ -0,0 +1,85 @@
+from typing import Any, Dict, List, Type, TypeVar, Union
+
+import attr
+
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="DebugOptions")
+
+
+@attr.s(auto_attribs=True)
+class DebugOptions:
+    """
+    Attributes:
+        raw_prompt (Union[Unset, None, str]): When `raw_prompt` is specified, it will be passed directly to the
+            inference engine for completion. `segments` field in `CompletionRequest` will be ignored.
+
+            This is useful for certain requests that aim to test the tabby's e2e quality.
+        return_snippets (Union[Unset, bool]): When true, returns `snippets` in `debug_data`.
+        return_prompt (Union[Unset, bool]): When true, returns `prompt` in `debug_data`.
+        disable_retrieval_augmented_code_completion (Union[Unset, bool]): When true, disable retrieval augmented code
+            completion.
+    """
+
+    raw_prompt: Union[Unset, None, str] = UNSET
+    return_snippets: Union[Unset, bool] = UNSET
+    return_prompt: Union[Unset, bool] = UNSET
+    disable_retrieval_augmented_code_completion: Union[Unset, bool] = UNSET
+    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        raw_prompt = self.raw_prompt
+        return_snippets = self.return_snippets
+        return_prompt = self.return_prompt
+        disable_retrieval_augmented_code_completion = self.disable_retrieval_augmented_code_completion
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+        if raw_prompt is not UNSET:
+            field_dict["raw_prompt"] = raw_prompt
+        if return_snippets is not UNSET:
+            field_dict["return_snippets"] = return_snippets
+        if return_prompt is not UNSET:
+            field_dict["return_prompt"] = return_prompt
+        if disable_retrieval_augmented_code_completion is not UNSET:
+            field_dict["disable_retrieval_augmented_code_completion"] = disable_retrieval_augmented_code_completion
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        raw_prompt = d.pop("raw_prompt", UNSET)
+
+        return_snippets = d.pop("return_snippets", UNSET)
+
+        return_prompt = d.pop("return_prompt", UNSET)
+
+        disable_retrieval_augmented_code_completion = d.pop("disable_retrieval_augmented_code_completion", UNSET)
+
+        debug_options = cls(
+            raw_prompt=raw_prompt,
+            return_snippets=return_snippets,
+            return_prompt=return_prompt,
+            disable_retrieval_augmented_code_completion=disable_retrieval_augmented_code_completion,
+        )
+
+        debug_options.additional_properties = d
+        return debug_options
+
+    @property
+    def additional_keys(self) -> List[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties