From b2a92f1cf77a93ac3747550ad56b79ab8ca8ed51 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Mon, 11 Dec 2023 12:40:41 +0800 Subject: [PATCH] test(golden): switch to cargo-insta for better developer experience (#1018) --- Cargo.lock | 32 +++++++++++++++ Cargo.toml | 4 ++ crates/tabby/Cargo.toml | 1 + crates/tabby/tests/golden.json | 35 ---------------- crates/tabby/tests/golden_chat.json | 24 ----------- crates/tabby/tests/goldentests.rs | 38 ++++++++++-------- crates/tabby/tests/goldentests_chat.rs | 40 ++++++++++++------- .../goldentests__assert_golden-2.snap | 10 +++++ .../snapshots/goldentests__assert_golden.snap | 10 +++++ .../goldentests__run_golden_tests-2.snap | 10 +++++ .../goldentests__run_golden_tests.snap | 10 +++++ .../goldentests_chat__assert_golden-2.snap | 6 +++ .../goldentests_chat__assert_golden.snap | 6 +++ 13 files changed, 136 insertions(+), 90 deletions(-) delete mode 100644 crates/tabby/tests/golden.json delete mode 100644 crates/tabby/tests/golden_chat.json create mode 100644 crates/tabby/tests/snapshots/goldentests__assert_golden-2.snap create mode 100644 crates/tabby/tests/snapshots/goldentests__assert_golden.snap create mode 100644 crates/tabby/tests/snapshots/goldentests__run_golden_tests-2.snap create mode 100644 crates/tabby/tests/snapshots/goldentests__run_golden_tests.snap create mode 100644 crates/tabby/tests/snapshots/goldentests_chat__assert_golden-2.snap create mode 100644 crates/tabby/tests/snapshots/goldentests_chat__assert_golden.snap diff --git a/Cargo.lock b/Cargo.lock index dc29f4516..9fd4dc01f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1611,6 +1611,22 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "insta" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "pest", + "pest_derive", + "serde", + "similar", + "yaml-rust", +] + [[package]] name = "instant" version = "0.1.12" @@ -3387,6 +3403,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -3599,6 +3621,7 @@ dependencies = [ "futures", "http-api-bindings", "hyper", + "insta", "lazy_static", "llama-cpp-bindings", "minijinja", @@ -5215,6 +5238,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index 28fcc1fec..1f0a493b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,3 +41,7 @@ utoipa = "3.3" axum = "0.6" hyper = "0.14" juniper = "0.15" + +[profile.dev.package] +insta.opt-level = 3 +similar.opt-level = 3 diff --git a/crates/tabby/Cargo.toml b/crates/tabby/Cargo.toml index d74b8ad08..5917b6f85 100644 --- a/crates/tabby/Cargo.toml +++ b/crates/tabby/Cargo.toml @@ -65,5 +65,6 @@ vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } [dev-dependencies] assert-json-diff = "2.0.2" +insta = { version = "1.34.0", features = ["yaml", "redactions"] } reqwest.workspace = true serde-jsonlines = "0.5.0" diff --git a/crates/tabby/tests/golden.json b/crates/tabby/tests/golden.json deleted file mode 100644 index 7f968d24b..000000000 --- a/crates/tabby/tests/golden.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "request": { - "language": "python", - "segments": { - "prefix": "def fib(n):\n ", - "suffix": "\n return fib(n - 1) + fib(n - 2)" - } - }, - "expected": { - "choices": [ - { - "index": 0, - "text": " if n <= 1:\n return n" - } - ] - } - }, - { - "request": { - "language": "python", - "segments": { - "prefix": "import datetime\n\ndef parse_expenses(expenses_string):\n \"\"\"Parse the list of expenses and return the list of triples (date, value, currency).\n Ignore lines starting with #.\n Parse the date using datetime.\n Example expenses_string:\n 2016-01-02 -34.01 USD\n 2016-01-03 2.59 DKK\n 2016-01-03 -2.72 EUR\n \"\"\"\n for line in expenses_string.split('\\n'):\n " - } - }, - "expected": { - "choices": [ - { - "index": 0, - "text": "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency" - } - ] - } - } -] \ No newline at end of file diff --git a/crates/tabby/tests/golden_chat.json b/crates/tabby/tests/golden_chat.json deleted file mode 100644 index e129c80b8..000000000 --- a/crates/tabby/tests/golden_chat.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "request": { - "messages": [ - { - "role": "user", - "content": "How to convert a list of string to numbers in python" - } - ] - }, - "expected": " In Python, you can convert a list of strings to numbers using the `map()` function and the `int()` function. Here's an example:\n```\nstrings = ['1', '2', '3', '4', '5']\nnumbers = list(map(int, strings))\nprint(numbers)\n```\nThis will output:\n```\n[1, 2, 3, 4, 5]\n```\nIn this example, the `map()` function applies the `int()` function to each element of the `strings` list, converting each string to an integer and returning a new list of integers. The `list()` function is used to convert the resulting iterator to a list." - }, - { - "request": { - "messages": [ - { - "role": "user", - "content": "How to parse email address with regex" - } - ] - }, - "expected": " To parse an email address with regex, you can use the following pattern:\n```\n^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$\n```\nThis pattern matches email addresses in the following format:\n\n* `^`: start of the string\n* `[a-zA-Z0-9._%+-]+`: matches one or more characters that are either letters, numbers, periods, underscores, percent signs, plus signs, or hyphens\n* `@`: matches the `@` symbol\n* `[a-zA-Z0-9.-]+`: matches one or more characters that are either letters, numbers, periods, or hyphens\n* `\\.`: matches the `.` symbol\n* `[a-zA-Z]{2,}`: matches two or more characters that are letters\n* `$`: end of the string\n\nYou can use this pattern in a programming language that supports regex, such as Python, JavaScript, or Java, to extract the email address from a string. For example, in Python, you can use the `re` module to find all email addresses in a string:\n```\nimport re\n\nstring = \"Please send your feedback to john.doe@example.com or jane_doe@example.co.uk.\"\n\nemails = re.findall(r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\", string)\n\nprint(emails) # Output: ['john.doe@example.com', 'jane_doe@example.co.uk']\n```\nIn this example, the `re.findall()` function is used to find all occurrences of the email address pattern in the string. The `findall()` function returns a list of all non-overlapping matches." - } -] \ No newline at end of file diff --git a/crates/tabby/tests/goldentests.rs b/crates/tabby/tests/goldentests.rs index 93edc9d97..af984c532 100644 --- a/crates/tabby/tests/goldentests.rs +++ b/crates/tabby/tests/goldentests.rs @@ -1,8 +1,7 @@ use std::path::PathBuf; -use assert_json_diff::assert_json_include; +use insta::assert_yaml_snapshot; use lazy_static::lazy_static; -use serde::Deserialize; use serde_json::json; use tokio::{ process::Command, @@ -48,10 +47,6 @@ fn tabby_path() -> PathBuf { workspace_dir().join("target/debug/tabby") } -fn golden_path() -> PathBuf { - workspace_dir().join("crates/tabby/tests/golden.json") -} - async fn wait_for_server() { lazy_static::initialize(&SERVER); @@ -68,7 +63,7 @@ async fn wait_for_server() { } } -async fn golden_test(body: serde_json::Value, expected: serde_json::Value) { +async fn golden_test(body: serde_json::Value) -> serde_json::Value { let mut body = body.clone(); body.as_object_mut().unwrap().insert( "debug_options".to_owned(), @@ -86,21 +81,32 @@ async fn golden_test(body: serde_json::Value, expected: serde_json::Value) { .json() .await .unwrap(); - assert_json_include!(actual: actual, expected: expected); + actual } -#[derive(Deserialize)] -struct TestCase { - request: serde_json::Value, - expected: serde_json::Value, +async fn assert_golden(body: serde_json::Value) { + assert_yaml_snapshot!(golden_test(body).await, { + ".id" => "test-id" + }); } #[tokio::test] async fn run_golden_tests() { wait_for_server().await; - let cases: Vec = serdeconv::from_json_file(golden_path()).unwrap(); - for case in cases { - golden_test(case.request, case.expected).await; - } + assert_golden(json!({ + "language": "python", + "segments": { + "prefix": "def fib(n):\n ", + "suffix": "\n return fib(n - 1) + fib(n - 2)" + } + })) + .await; + + assert_golden(json!({ + "language": "python", + "segments": { + "prefix": "import datetime\n\ndef parse_expenses(expenses_string):\n \"\"\"Parse the list of expenses and return the list of triples (date, value, currency).\n Ignore lines starting with #.\n Parse the date using datetime.\n Example expenses_string:\n 2016-01-02 -34.01 USD\n 2016-01-03 2.59 DKK\n 2016-01-03 -2.72 EUR\n \"\"\"\n for line in expenses_string.split('\\n'):\n " + } + })).await; } diff --git a/crates/tabby/tests/goldentests_chat.rs b/crates/tabby/tests/goldentests_chat.rs index ff54c88c8..86e982636 100644 --- a/crates/tabby/tests/goldentests_chat.rs +++ b/crates/tabby/tests/goldentests_chat.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; -use assert_json_diff::assert_json_include; +use insta::assert_yaml_snapshot; use lazy_static::lazy_static; use serde::Deserialize; +use serde_json::json; use serde_jsonlines::BufReadExt; use tokio::{ process::Command, @@ -53,10 +54,6 @@ fn tabby_path() -> PathBuf { workspace_dir().join("target/debug/tabby") } -fn golden_path() -> PathBuf { - workspace_dir().join("crates/tabby/tests/golden_chat.json") -} - async fn wait_for_server() { lazy_static::initialize(&SERVER); @@ -73,7 +70,7 @@ async fn wait_for_server() { } } -async fn golden_test(body: serde_json::Value, expected: serde_json::Value) { +async fn golden_test(body: serde_json::Value) -> String { let bytes = CLIENT .post("http://localhost:9090/v1beta/chat/completions") .json(&body) @@ -90,21 +87,34 @@ async fn golden_test(body: serde_json::Value, expected: serde_json::Value) { actual += &x.unwrap().content; } - assert_json_include!(actual: actual, expected: expected); + actual } -#[derive(Deserialize)] -struct TestCase { - request: serde_json::Value, - expected: serde_json::Value, +async fn assert_golden(body: serde_json::Value) { + assert_yaml_snapshot!(golden_test(body).await); } #[tokio::test] async fn run_chat_golden_tests() { wait_for_server().await; - let cases: Vec = serdeconv::from_json_file(golden_path()).unwrap(); - for case in cases { - golden_test(case.request, case.expected).await; - } + assert_golden(json!({ + "messages": [ + { + "role": "user", + "content": "How to convert a list of string to numbers in python" + } + ] + })) + .await; + + assert_golden(json!({ + "messages": [ + { + "role": "user", + "content": "How to parse email address with regex" + } + ] + })) + .await; } diff --git a/crates/tabby/tests/snapshots/goldentests__assert_golden-2.snap b/crates/tabby/tests/snapshots/goldentests__assert_golden-2.snap new file mode 100644 index 000000000..051ed06b0 --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests__assert_golden-2.snap @@ -0,0 +1,10 @@ +--- +source: crates/tabby/tests/goldentests.rs +expression: golden_test(body).await +--- +id: test-id +choices: + - index: 0 + text: "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency" +debug_data: {} + diff --git a/crates/tabby/tests/snapshots/goldentests__assert_golden.snap b/crates/tabby/tests/snapshots/goldentests__assert_golden.snap new file mode 100644 index 000000000..445fb4679 --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests__assert_golden.snap @@ -0,0 +1,10 @@ +--- +source: crates/tabby/tests/goldentests.rs +expression: golden_test(body).await +--- +id: test-id +choices: + - index: 0 + text: " if n <= 1:\n return n" +debug_data: {} + diff --git a/crates/tabby/tests/snapshots/goldentests__run_golden_tests-2.snap b/crates/tabby/tests/snapshots/goldentests__run_golden_tests-2.snap new file mode 100644 index 000000000..ad49159d2 --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests__run_golden_tests-2.snap @@ -0,0 +1,10 @@ +--- +source: crates/tabby/tests/goldentests.rs +expression: "golden_test(json!({\n \"language\" : \"python\", \"segments\" :\n {\n \"prefix\" :\n \"import datetime\\n\\ndef parse_expenses(expenses_string):\\n \\\"\\\"\\\"Parse the list of expenses and return the list of triples (date, value, currency).\\n Ignore lines starting with #.\\n Parse the date using datetime.\\n Example expenses_string:\\n 2016-01-02 -34.01 USD\\n 2016-01-03 2.59 DKK\\n 2016-01-03 -2.72 EUR\\n \\\"\\\"\\\"\\n for line in expenses_string.split('\\\\n'):\\n \"\n }\n })).await" +--- +id: cmpl-9e7f9be8-3bf3-4d90-9a5f-05067784a35f +choices: + - index: 0 + text: "if line.startswith('#'):\n continue\n date, value, currency = line.split()\n date = datetime.datetime.strptime(date, '%Y-%m-%d')\n yield date, float(value), currency" +debug_data: {} + diff --git a/crates/tabby/tests/snapshots/goldentests__run_golden_tests.snap b/crates/tabby/tests/snapshots/goldentests__run_golden_tests.snap new file mode 100644 index 000000000..f6e5d3827 --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests__run_golden_tests.snap @@ -0,0 +1,10 @@ +--- +source: crates/tabby/tests/goldentests.rs +expression: "golden_test(json!({\n \"language\" : \"python\", \"segments\" :\n {\n \"prefix\" : \"def fib(n):\\n \", \"suffix\" :\n \"\\n return fib(n - 1) + fib(n - 2)\"\n }\n })).await" +--- +id: cmpl-9abdf3d1-11f7-4f26-96af-0c4175d9bb53 +choices: + - index: 0 + text: " if n <= 1:\n return n" +debug_data: {} + diff --git a/crates/tabby/tests/snapshots/goldentests_chat__assert_golden-2.snap b/crates/tabby/tests/snapshots/goldentests_chat__assert_golden-2.snap new file mode 100644 index 000000000..b8f859a26 --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests_chat__assert_golden-2.snap @@ -0,0 +1,6 @@ +--- +source: crates/tabby/tests/goldentests_chat.rs +expression: golden_test(body).await +--- +" To parse an email address with regex, you can use the following pattern:\n```\n^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$\n```\nThis pattern matches email addresses in the following format:\n\n* `^`: start of the string\n* `[a-zA-Z0-9._%+-]+`: matches one or more characters that are either letters, numbers, periods, underscores, percent signs, plus signs, or hyphens\n* `@`: matches the `@` symbol\n* `[a-zA-Z0-9.-]+`: matches one or more characters that are either letters, numbers, periods, or hyphens\n* `\\.`: matches the `.` symbol\n* `[a-zA-Z]{2,}`: matches two or more characters that are letters\n* `$`: end of the string\n\nYou can use this pattern in a programming language that supports regex, such as Python, JavaScript, or Java, to extract the email address from a string. For example, in Python, you can use the `re` module to find all email addresses in a string:\n```\nimport re\n\nstring = \"Please send your feedback to john.doe@example.com or jane_doe@example.co.uk.\"\n\nemails = re.findall(r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\", string)\n\nprint(emails) # Output: ['john.doe@example.com', 'jane_doe@example.co.uk']\n```\nIn this example, the `re.findall()` function is used to find all occurrences of the email address pattern in the string. The `findall()` function returns a list of all non-overlapping matches." + diff --git a/crates/tabby/tests/snapshots/goldentests_chat__assert_golden.snap b/crates/tabby/tests/snapshots/goldentests_chat__assert_golden.snap new file mode 100644 index 000000000..5d8f2666f --- /dev/null +++ b/crates/tabby/tests/snapshots/goldentests_chat__assert_golden.snap @@ -0,0 +1,6 @@ +--- +source: crates/tabby/tests/goldentests_chat.rs +expression: golden_test(body).await +--- +" In Python, you can convert a list of strings to numbers using the `map()` function and the `int()` function. Here's an example:\n```\nstrings = ['1', '2', '3', '4', '5']\nnumbers = list(map(int, strings))\nprint(numbers)\n```\nThis will output:\n```\n[1, 2, 3, 4, 5]\n```\nIn this example, the `map()` function applies the `int()` function to each element of the `strings` list, converting each string to an integer and returning a new list of integers. The `list()` function is used to convert the resulting iterator to a list." +