chore: add experimental eval setup

experimental/eval/docker-compose.yaml

@@ -0,0 +1,19 @@
+version: '3.5'
+
+services:
+  scheduler:
+    image: tabbyml/tabby
+    platform: linux/amd64
+    command: scheduler --now
+    volumes:
+      - "$HOME/.tabby:/data"
+
+  tabby:
+    depends_on:
+      - scheduler
+    restart: always
+    image: tabbyml/tabby
+    platform: linux/amd64
+    command: serve --model TabbyML/T5P-220M
+    volumes:
+      - "$HOME/.tabby:/data"

experimental/eval/skypilot.yaml

@@ -0,0 +1,24 @@
+resources:
+  accelerators: T4:1
+
+setup: |
+  set -ex
+
+  # On some cloud providers, docker-compose is not installed by default.
+  sudo curl -L https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
+  sudo chmod a+x /usr/local/bin/docker-compose
+
+  # Pull tabby images.
+  git clone https://github.com/TabbyML/tabby.git || true
+  cd tabby/experimental
+
+  # On certain cloud providers (e.g lambda cloud), the default user is not added to docker group, so we need sudo here
+  sudo docker-compose pull
+
+  # Add current user to docker group, it won't take effect immediately as skypilot job is started by a long-running daemon.
+  sudo usermod -aG docker $USER
+
+
+run: |
+  cd tabby/experimental
+  sudo docker-compose up