mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly synced 2024-11-22 07:33:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

wire json::Path into json_family (#2587)

Browse Source 
* feat: wire json::Path into the server

The feature is disabled under a flag and it covers read operations for now.
---------

Signed-off-by: Roman Gershman <roman@dragonflydb.io>
This commit is contained in:
Roman Gershman 2024-02-15 19:05:53 +02:00 committed by GitHub
parent 1e06c63727
commit 888a437bf5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 117 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -8,6 +8,7 @@ clang-*
third_party
genfiles/*
*.sublime-*
*.orig
.tags
!third_party/include/*
*.pyc

15
src/core/json/jsonpath_test.cc
View File

@ -249,4 +249,19 @@ TEST_F(JsonPathTest, EvalDescent) {
ASSERT_THAT(arr, ElementsAre(json_type::array_value, json_type::object_value));
}
TEST_F(JsonPathTest, Wildcard) {
ASSERT_EQ(0, Parse("$[*]"));
Path path = driver_.TakePath();
ASSERT_EQ(1, path.size());
EXPECT_THAT(path[0], SegType(SegmentType::WILDCARD));
JsonType json = JsonFromString(R"([1, 2, 3])").value();
vector<int> arr;
EvaluatePath(path, json, [&](optional<string_view> key, const JsonType& val) {
ASSERT_FALSE(key);
arr.push_back(val.as<int>());
});
ASSERT_THAT(arr, ElementsAre(1, 2, 3));
}
} // namespace dfly::json

2
src/core/json/path.cc
View File

@ -152,7 +152,7 @@ auto Dfs::Item::Init(const PathSegment& segment) -> AdvanceResult {
if (segment_step_ == 1) {
// first time, branching to return the same object but with the next segment,
// exploring the path of ignoring the DESCENT operator.
// Alsom, shift the state (segment_step) to bypass this branch next time.
// Also, shift the state (segment_step) to bypass this branch next time.
segment_step_ = 0;
return DepthState{depth_state_.first, depth_state_.second + 1};
}

6
src/facade/facade_test.h
View File

@ -78,9 +78,9 @@ inline ::testing::PolymorphicMatcher<RespTypeMatcher> ArgType(RespExpr::Type t)
}
MATCHER_P(RespArray, value, "") {
return ExplainMatchResult(testing::AllOf(testing::Field(&RespExpr::type, RespExpr::ARRAY),
testing::Property(&RespExpr::GetVec, value)),
arg, result_listener);
return ExplainMatchResult(
testing::AllOf(ArgType(RespExpr::ARRAY), testing::Property(&RespExpr::GetVec, value)), arg,
result_listener);
}
inline bool operator==(const RespExpr& left, std::string_view s) {

3
src/server/CMakeLists.txt
View File

@ -62,7 +62,8 @@ endif()
find_library(ZSTD_LIB NAMES libzstd.a libzstdstatic.a zstd NAMES_PER_DIR REQUIRED)
cxx_link(dfly_transaction dfly_core strings_lib TRDP::fast_float)
cxx_link(dragonfly_lib dfly_transaction dfly_facade redis_lib awsv2_lib strings_lib html_lib
cxx_link(dragonfly_lib dfly_transaction dfly_facade redis_lib awsv2_lib jsonpath
strings_lib html_lib
http_client_lib absl::random_random TRDP::jsoncons ${ZSTD_LIB} TRDP::lz4
TRDP::croncpp)

125
src/server/json_family.cc
View File

@ -19,7 +19,10 @@ extern "C" {
#include <jsoncons_ext/jsonpath/jsonpath.hpp>
#include <jsoncons_ext/jsonpointer/jsonpointer.hpp>
#include "base/flags.h"
#include "base/logging.h"
#include "core/json/driver.h"
#include "core/json/jsonpath_grammar.hh"
#include "core/json_object.h"
#include "facade/cmd_arg_parser.h"
#include "server/acl/acl_commands_def.h"
@ -30,10 +33,13 @@ extern "C" {
#include "server/tiered_storage.h"
#include "server/transaction.h"
ABSL_FLAG(bool, jsonpathv2, false, "If true uses Dragonfly jsonpath implementation.");
namespace dfly {
using namespace std;
using namespace jsoncons;
using facade::kSyntaxErrType;
using JsonExpression = jsonpath::jsonpath_expression<JsonType>;
using OptBool = optional<bool>;
@ -48,6 +54,42 @@ static const char DefaultJsonPath[] = "$";
namespace {
using JsonPathV2 = variant<json::Path, JsonExpression>;
using ExprCallback = absl::FunctionRef<void(string_view, const JsonType&)>;
inline void Evaluate(const JsonExpression& expr, const JsonType& obj, ExprCallback cb) {
expr.evaluate(obj, cb);
}
inline void Evaluate(const json::Path& expr, const JsonType& obj, ExprCallback cb) {
json::EvaluatePath(expr, obj, [&cb](optional<string_view> key, const JsonType& val) {
cb(key ? *key : string_view{}, val);
});
}
class JsonPathDriver : public json::Driver {
public:
string msg;
void Error(const json::location& l, const std::string& msg) final {
this->msg = absl::StrCat("Error: ", msg);
}
};
io::Result<json::Path, string> JsonPathV2Parse(string_view path) {
if (path.size() > 8_KB)
return nonstd::make_unexpected("Path too long");
JsonPathDriver driver;
json::Parser parser(&driver);
driver.SetInput(string(path));
int res = parser();
if (res != 0) {
return nonstd::make_unexpected(driver.msg);
}
return driver.TakePath();
}
inline OpStatus JsonReplaceVerifyNoOp(JsonType&) {
return OpStatus::OK;
}
@ -457,7 +499,7 @@ OpResult<string> OpJsonGet(const OpArgs& op_args, string_view key,
return out.as<string>();
}
OpResult<vector<string>> OpType(const OpArgs& op_args, string_view key, JsonExpression expression) {
OpResult<vector<string>> OpType(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -469,12 +511,11 @@ OpResult<vector<string>> OpType(const OpArgs& op_args, string_view key, JsonExpr
vec.emplace_back(JsonTypeToName(val));
};
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
OpResult<vector<OptSizeT>> OpStrLen(const OpArgs& op_args, string_view key,
JsonExpression expression) {
OpResult<vector<OptSizeT>> OpStrLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -489,12 +530,11 @@ OpResult<vector<OptSizeT>> OpStrLen(const OpArgs& op_args, string_view key,
}
};
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
OpResult<vector<OptSizeT>> OpObjLen(const OpArgs& op_args, string_view key,
JsonExpression expression) {
OpResult<vector<OptSizeT>> OpObjLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -510,12 +550,11 @@ OpResult<vector<OptSizeT>> OpObjLen(const OpArgs& op_args, string_view key,
}
};
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
OpResult<vector<OptSizeT>> OpArrLen(const OpArgs& op_args, string_view key,
JsonExpression expression) {
OpResult<vector<OptSizeT>> OpArrLen(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -531,7 +570,7 @@ OpResult<vector<OptSizeT>> OpArrLen(const OpArgs& op_args, string_view key,
}
};
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
@ -649,7 +688,7 @@ OpResult<long> OpDel(const OpArgs& op_args, string_view key, string_view path) {
// Returns a vector of string vectors,
// keys within the same object are stored in the same string vector.
OpResult<vector<StringVec>> OpObjKeys(const OpArgs& op_args, string_view key,
JsonExpression expression) {
JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -658,6 +697,8 @@ OpResult<vector<StringVec>> OpObjKeys(const OpArgs& op_args, string_view key,
vector<StringVec> vec;
auto cb = [&vec](const string_view& path, const JsonType& val) {
// Aligned with ElastiCache flavor.
DVLOG(2) << "path: " << path << " val: " << val.to_string();
if (!val.is_object()) {
vec.emplace_back();
return;
@ -669,8 +710,8 @@ OpResult<vector<StringVec>> OpObjKeys(const OpArgs& op_args, string_view key,
}
};
JsonType& json_entry = *(result.value());
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
expression.evaluate(json_entry, cb);
return vec;
}
@ -917,9 +958,8 @@ OpResult<vector<OptSizeT>> OpArrAppend(const OpArgs& op_args, string_view key, s
// Returns a numeric vector representing each JSON value first index of the JSON scalar.
// An index value of -1 represents unfound in the array.
// JSON scalar has types of string, boolean, null, and number.
OpResult<vector<OptLong>> OpArrIndex(const OpArgs& op_args, string_view key,
JsonExpression expression, const JsonType& search_val,
int start_index, int end_index) {
OpResult<vector<OptLong>> OpArrIndex(const OpArgs& op_args, string_view key, JsonPathV2 expression,
const JsonType& search_val, int start_index, int end_index) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -975,12 +1015,12 @@ OpResult<vector<OptLong>> OpArrIndex(const OpArgs& op_args, string_view key,
vec.emplace_back(pos);
};
JsonType& json_entry = *(result.value());
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
// Returns string vector that represents the query result of each supplied key.
vector<OptString> OpJsonMGet(JsonExpression expression, const Transaction* t, EngineShard* shard) {
vector<OptString> OpJsonMGet(JsonPathV2 expression, const Transaction* t, EngineShard* shard) {
auto args = t->GetShardArgs(shard->shard_id());
DCHECK(!args.empty());
vector<OptString> response(args.size());
@ -1002,7 +1042,7 @@ vector<OptString> OpJsonMGet(JsonExpression expression, const Transaction* t, En
};
const JsonType& json_entry = *(json_val);
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
if (query_result.empty()) {
continue;
@ -1028,8 +1068,7 @@ vector<OptString> OpJsonMGet(JsonExpression expression, const Transaction* t, En
}
// Returns numeric vector that represents the number of fields of JSON value at each path.
OpResult<vector<OptSizeT>> OpFields(const OpArgs& op_args, string_view key,
JsonExpression expression) {
OpResult<vector<OptSizeT>> OpFields(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -1040,13 +1079,12 @@ OpResult<vector<OptSizeT>> OpFields(const OpArgs& op_args, string_view key,
vec.emplace_back(CountJsonFields(val));
};
const JsonType& json_entry = *(result.value());
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
// Returns json vector that represents the result of the json query.
OpResult<vector<JsonType>> OpResp(const OpArgs& op_args, string_view key,
JsonExpression expression) {
OpResult<vector<JsonType>> OpResp(const OpArgs& op_args, string_view key, JsonPathV2 expression) {
OpResult<JsonType*> result = GetJson(op_args, key);
if (!result) {
return result.status();
@ -1055,7 +1093,7 @@ OpResult<vector<JsonType>> OpResp(const OpArgs& op_args, string_view key,
vector<JsonType> vec;
auto cb = [&vec](const string_view& path, const JsonType& val) { vec.emplace_back(val); };
const JsonType& json_entry = *(result.value());
expression.evaluate(json_entry, cb);
visit([&](auto&& arg) { Evaluate(arg, json_entry, cb); }, expression);
return vec;
}
@ -1141,6 +1179,17 @@ OpResult<bool> OpSet(const OpArgs& op_args, string_view key, string_view path,
return operation_result;
}
io::Result<JsonPathV2, string> ParsePathV2(string_view path) {
if (absl::GetFlag(FLAGS_jsonpathv2)) {
return JsonPathV2Parse(path);
}
io::Result<JsonExpression> expr_result = ParseJsonPath(path);
if (!expr_result) {
return nonstd::make_unexpected(kSyntaxErr);
}
return JsonPathV2(std::move(expr_result.value()));
}
} // namespace
// GCC extension of returning a value of multiple statements. The last statement is returned.
@ -1155,6 +1204,16 @@ OpResult<bool> OpSet(const OpArgs& op_args, string_view key, string_view path,
std::move(*expr_result); \
})
#define PARSE_PATHV2(path) \
({ \
auto result = ParsePathV2(path); \
if (!result) { \
cntx->SendError(result.error()); \
return; \
} \
std::move(*result); \
})
void JsonFamily::Set(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
@ -1201,7 +1260,7 @@ void JsonFamily::Resp(CmdArgList args, ConnectionContext* cntx) {
path = ArgS(args, 1);
}
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpResp(t->GetOpArgs(shard), key, std::move(expression));
@ -1269,7 +1328,7 @@ void JsonFamily::MGet(CmdArgList args, ConnectionContext* cntx) {
DCHECK_GE(args.size(), 1U);
string_view path = ArgS(args, args.size() - 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
Transaction* transaction = cntx->transaction;
unsigned shard_count = shard_set->size();
@ -1319,7 +1378,7 @@ void JsonFamily::ArrIndex(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
optional<JsonType> search_value = JsonFromString(ArgS(args, 2));
if (!search_value) {
@ -1542,7 +1601,7 @@ void JsonFamily::ObjKeys(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpObjKeys(t->GetOpArgs(shard), key, std::move(expression));
@ -1653,7 +1712,7 @@ void JsonFamily::Type(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpType(t->GetOpArgs(shard), key, std::move(expression));
@ -1682,7 +1741,7 @@ void JsonFamily::ArrLen(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpArrLen(t->GetOpArgs(shard), key, std::move(expression));
@ -1702,7 +1761,7 @@ void JsonFamily::ObjLen(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpObjLen(t->GetOpArgs(shard), key, std::move(expression));
@ -1722,7 +1781,7 @@ void JsonFamily::StrLen(CmdArgList args, ConnectionContext* cntx) {
string_view key = ArgS(args, 0);
string_view path = ArgS(args, 1);
JsonExpression expression = PARSE_PATH_ARG(path);
JsonPathV2 expression = PARSE_PATHV2(path);
auto cb = [&](Transaction* t, EngineShard* shard) {
return OpStrLen(t->GetOpArgs(shard), key, std::move(expression));

7
src/server/json_family_test.cc
View File

@ -165,9 +165,8 @@ TEST_F(JsonFamilyTest, Type) {
ASSERT_THAT(resp, "OK");
resp = Run({"JSON.TYPE", "json", "$[*]"});
ASSERT_EQ(RespExpr::ARRAY, resp.type);
EXPECT_THAT(resp.GetVec(),
ElementsAre("integer", "number", "string", "boolean", "null", "object", "array"));
ASSERT_THAT(resp, RespArray(ElementsAre("integer", "number", "string", "boolean", "null",
"object", "array")));
resp = Run({"JSON.TYPE", "json", "$[10]"});
EXPECT_THAT(resp, ArgType(RespExpr::NIL));
@ -589,7 +588,7 @@ TEST_F(JsonFamilyTest, ObjKeys) {
ASSERT_THAT(resp, ArrLen(2));
const auto& arr1 = resp.GetVec();
EXPECT_THAT(arr1[0], ArgType(RespExpr::NIL_ARRAY));
EXPECT_THAT(arr1[1].GetVec(), ElementsAre("b", "c"));
EXPECT_THAT(arr1[1], RespArray(ElementsAre("b", "c")));
json = R"(
{"a":{}, "b":{"c":{"d": {"e": 1337}}}}