mirrors/tabby
1
0
Fork 0
mirror of https://github.com/TabbyML/tabby synced 2024-11-22 17:41:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(golden): switch to cargo-insta for better developer experience (#1018)

Browse Source
This commit is contained in:
Meng Zhang 2023-12-11 12:40:41 +08:00 committed by GitHub
parent e0d0133d86
commit b2a92f1cf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 136 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@ -1611,6 +1611,22 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "insta"
version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc"
dependencies = [
"console",
"lazy_static",
"linked-hash-map",
"pest",
"pest_derive",
"serde",
"similar",
"yaml-rust",
]
[[package]]
name = "instant"
version = "0.1.12"
@ -3387,6 +3403,12 @@ dependencies = [
"libc",
]
[[package]]
name = "similar"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597"
[[package]]
name = "simple_asn1"
version = "0.6.2"
@ -3599,6 +3621,7 @@ dependencies = [
"futures",
"http-api-bindings",
"hyper",
"insta",
"lazy_static",
"llama-cpp-bindings",
"minijinja",
@ -5215,6 +5238,15 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "zip"
version = "0.6.6"

4
Cargo.toml
View File

@ -41,3 +41,7 @@ utoipa = "3.3"
axum = "0.6"
hyper = "0.14"
juniper = "0.15"
[profile.dev.package]
insta.opt-level = 3
similar.opt-level = 3

1
crates/tabby/Cargo.toml
View File

@ -65,5 +65,6 @@ vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] }
[dev-dependencies]
assert-json-diff = "2.0.2"
insta = { version = "1.34.0", features = ["yaml", "redactions"] }
reqwest.workspace = true
serde-jsonlines = "0.5.0"

35
crates/tabby/tests/golden.json
View File

@ -1,35 +0,0 @@
[
{
"request": {
"language": "python",
"segments": {
"prefix": "def fib(n):\n ",
"suffix": "\n return fib(n - 1) + fib(n - 2)"
}
},
"expected": {
"choices": [
{
"index": 0,
"text": " if n <= 1:\n return n"
}
]
}
},
{
"request": {
"language": "python",
"segments": {
"prefix": "import datetime\n\ndef parse_expenses(expenses_string):\n \"\"\"Parse the list of expenses and return the list of triples (date, value, currency).\n Ignore lines starting with #.\n Parse the date using datetime.\n Example expenses_string:\n 2016-01-02 -34.01 USD\n 2016-01-03 2.59 DKK\n 2016-01-03 -2.72 EUR\n \"\"\"\n for line in expenses_string.split('\\n'):\n "
}
},
"expected": {
"choices": [
{
"index": 0,
"text": "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency"
}
]
}
}
]

24
crates/tabby/tests/golden_chat.json
View File

@ -1,24 +0,0 @@
[
{
"request": {
"messages": [
{
"role": "user",
"content": "How to convert a list of string to numbers in python"
}
]
},
"expected": " In Python, you can convert a list of strings to numbers using the `map()` function and the `int()` function. Here's an example:\n```\nstrings = ['1', '2', '3', '4', '5']\nnumbers = list(map(int, strings))\nprint(numbers)\n```\nThis will output:\n```\n[1, 2, 3, 4, 5]\n```\nIn this example, the `map()` function applies the `int()` function to each element of the `strings` list, converting each string to an integer and returning a new list of integers. The `list()` function is used to convert the resulting iterator to a list."
},
{
"request": {
"messages": [
{
"role": "user",
"content": "How to parse email address with regex"
}
]
},
"expected": " To parse an email address with regex, you can use the following pattern:\n```\n^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$\n```\nThis pattern matches email addresses in the following format:\n\n* `^`: start of the string\n* `[a-zA-Z0-9._%+-]+`: matches one or more characters that are either letters, numbers, periods, underscores, percent signs, plus signs, or hyphens\n* `@`: matches the `@` symbol\n* `[a-zA-Z0-9.-]+`: matches one or more characters that are either letters, numbers, periods, or hyphens\n* `\\.`: matches the `.` symbol\n* `[a-zA-Z]{2,}`: matches two or more characters that are letters\n* `$`: end of the string\n\nYou can use this pattern in a programming language that supports regex, such as Python, JavaScript, or Java, to extract the email address from a string. For example, in Python, you can use the `re` module to find all email addresses in a string:\n```\nimport re\n\nstring = \"Please send your feedback to john.doe@example.com or jane_doe@example.co.uk.\"\n\nemails = re.findall(r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\", string)\n\nprint(emails) # Output: ['john.doe@example.com', 'jane_doe@example.co.uk']\n```\nIn this example, the `re.findall()` function is used to find all occurrences of the email address pattern in the string. The `findall()` function returns a list of all non-overlapping matches."
}
]

38
crates/tabby/tests/goldentests.rs
View File

@ -1,8 +1,7 @@
use std::path::PathBuf;
use assert_json_diff::assert_json_include;
use insta::assert_yaml_snapshot;
use lazy_static::lazy_static;
use serde::Deserialize;
use serde_json::json;
use tokio::{
process::Command,
@ -48,10 +47,6 @@ fn tabby_path() -> PathBuf {
workspace_dir().join("target/debug/tabby")
}
fn golden_path() -> PathBuf {
workspace_dir().join("crates/tabby/tests/golden.json")
}
async fn wait_for_server() {
lazy_static::initialize(&SERVER);
@ -68,7 +63,7 @@ async fn wait_for_server() {
}
}
async fn golden_test(body: serde_json::Value, expected: serde_json::Value) {
async fn golden_test(body: serde_json::Value) -> serde_json::Value {
let mut body = body.clone();
body.as_object_mut().unwrap().insert(
"debug_options".to_owned(),
@ -86,21 +81,32 @@ async fn golden_test(body: serde_json::Value, expected: serde_json::Value) {
.json()
.await
.unwrap();
assert_json_include!(actual: actual, expected: expected);
actual
}
#[derive(Deserialize)]
struct TestCase {
request: serde_json::Value,
expected: serde_json::Value,
async fn assert_golden(body: serde_json::Value) {
assert_yaml_snapshot!(golden_test(body).await, {
".id" => "test-id"
});
}
#[tokio::test]
async fn run_golden_tests() {
wait_for_server().await;
let cases: Vec<TestCase> = serdeconv::from_json_file(golden_path()).unwrap();
for case in cases {
golden_test(case.request, case.expected).await;
}
assert_golden(json!({
"language": "python",
"segments": {
"prefix": "def fib(n):\n ",
"suffix": "\n return fib(n - 1) + fib(n - 2)"
}
}))
.await;
assert_golden(json!({
"language": "python",
"segments": {
"prefix": "import datetime\n\ndef parse_expenses(expenses_string):\n \"\"\"Parse the list of expenses and return the list of triples (date, value, currency).\n Ignore lines starting with #.\n Parse the date using datetime.\n Example expenses_string:\n 2016-01-02 -34.01 USD\n 2016-01-03 2.59 DKK\n 2016-01-03 -2.72 EUR\n \"\"\"\n for line in expenses_string.split('\\n'):\n "
}
})).await;
}

40
crates/tabby/tests/goldentests_chat.rs
View File

@ -1,8 +1,9 @@
use std::path::PathBuf;
use assert_json_diff::assert_json_include;
use insta::assert_yaml_snapshot;
use lazy_static::lazy_static;
use serde::Deserialize;
use serde_json::json;
use serde_jsonlines::BufReadExt;
use tokio::{
process::Command,
@ -53,10 +54,6 @@ fn tabby_path() -> PathBuf {
workspace_dir().join("target/debug/tabby")
}
fn golden_path() -> PathBuf {
workspace_dir().join("crates/tabby/tests/golden_chat.json")
}
async fn wait_for_server() {
lazy_static::initialize(&SERVER);
@ -73,7 +70,7 @@ async fn wait_for_server() {
}
}
async fn golden_test(body: serde_json::Value, expected: serde_json::Value) {
async fn golden_test(body: serde_json::Value) -> String {
let bytes = CLIENT
.post("http://localhost:9090/v1beta/chat/completions")
.json(&body)
@ -90,21 +87,34 @@ async fn golden_test(body: serde_json::Value, expected: serde_json::Value) {
actual += &x.unwrap().content;
}
assert_json_include!(actual: actual, expected: expected);
actual
}
#[derive(Deserialize)]
struct TestCase {
request: serde_json::Value,
expected: serde_json::Value,
async fn assert_golden(body: serde_json::Value) {
assert_yaml_snapshot!(golden_test(body).await);
}
#[tokio::test]
async fn run_chat_golden_tests() {
wait_for_server().await;
let cases: Vec<TestCase> = serdeconv::from_json_file(golden_path()).unwrap();
for case in cases {
golden_test(case.request, case.expected).await;
}
assert_golden(json!({
"messages": [
{
"role": "user",
"content": "How to convert a list of string to numbers in python"
}
]
}))
.await;
assert_golden(json!({
"messages": [
{
"role": "user",
"content": "How to parse email address with regex"
}
]
}))
.await;
}

10
crates/tabby/tests/snapshots/goldentests__assert_golden-2.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/tabby/tests/goldentests.rs
expression: golden_test(body).await
---
id: test-id
choices:
- index: 0
text: "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency"
debug_data: {}

10
crates/tabby/tests/snapshots/goldentests__assert_golden.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/tabby/tests/goldentests.rs
expression: golden_test(body).await
---
id: test-id
choices:
- index: 0
text: " if n <= 1:\n return n"
debug_data: {}

10
crates/tabby/tests/snapshots/goldentests__run_golden_tests-2.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/tabby/tests/goldentests.rs
expression: "golden_test(json!({\n \"language\" : \"python\", \"segments\" :\n {\n \"prefix\" :\n \"import datetime\\n\\ndef parse_expenses(expenses_string):\\n \\\"\\\"\\\"Parse the list of expenses and return the list of triples (date, value, currency).\\n Ignore lines starting with #.\\n Parse the date using datetime.\\n Example expenses_string:\\n 2016-01-02 -34.01 USD\\n 2016-01-03 2.59 DKK\\n 2016-01-03 -2.72 EUR\\n \\\"\\\"\\\"\\n for line in expenses_string.split('\\\\n'):\\n \"\n }\n })).await"
---
id: cmpl-9e7f9be8-3bf3-4d90-9a5f-05067784a35f
choices:
- index: 0
text: "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency"
debug_data: {}

10
crates/tabby/tests/snapshots/goldentests__run_golden_tests.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/tabby/tests/goldentests.rs
expression: "golden_test(json!({\n \"language\" : \"python\", \"segments\" :\n {\n \"prefix\" : \"def fib(n):\\n \", \"suffix\" :\n \"\\n return fib(n - 1) + fib(n - 2)\"\n }\n })).await"
---
id: cmpl-9abdf3d1-11f7-4f26-96af-0c4175d9bb53
choices:
- index: 0
text: " if n <= 1:\n return n"
debug_data: {}

6
crates/tabby/tests/snapshots/goldentests_chat__assert_golden-2.snap Normal file
View File

@ -0,0 +1,6 @@
---
source: crates/tabby/tests/goldentests_chat.rs
expression: golden_test(body).await
---
" To parse an email address with regex, you can use the following pattern:\n```\n^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$\n```\nThis pattern matches email addresses in the following format:\n\n* `^`: start of the string\n* `[a-zA-Z0-9._%+-]+`: matches one or more characters that are either letters, numbers, periods, underscores, percent signs, plus signs, or hyphens\n* `@`: matches the `@` symbol\n* `[a-zA-Z0-9.-]+`: matches one or more characters that are either letters, numbers, periods, or hyphens\n* `\\.`: matches the `.` symbol\n* `[a-zA-Z]{2,}`: matches two or more characters that are letters\n* `$`: end of the string\n\nYou can use this pattern in a programming language that supports regex, such as Python, JavaScript, or Java, to extract the email address from a string. For example, in Python, you can use the `re` module to find all email addresses in a string:\n```\nimport re\n\nstring = \"Please send your feedback to john.doe@example.com or jane_doe@example.co.uk.\"\n\nemails = re.findall(r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\", string)\n\nprint(emails) # Output: ['john.doe@example.com', 'jane_doe@example.co.uk']\n```\nIn this example, the `re.findall()` function is used to find all occurrences of the email address pattern in the string. The `findall()` function returns a list of all non-overlapping matches."

6
crates/tabby/tests/snapshots/goldentests_chat__assert_golden.snap Normal file
View File

@ -0,0 +1,6 @@
---
source: crates/tabby/tests/goldentests_chat.rs
expression: golden_test(body).await
---
" In Python, you can convert a list of strings to numbers using the `map()` function and the `int()` function. Here's an example:\n```\nstrings = ['1', '2', '3', '4', '5']\nnumbers = list(map(int, strings))\nprint(numbers)\n```\nThis will output:\n```\n[1, 2, 3, 4, 5]\n```\nIn this example, the `map()` function applies the `int()` function to each element of the `strings` list, converting each string to an integer and returning a new list of integers. The `list()` function is used to convert the resulting iterator to a list."