From c2a111ca1952be5553ed5eda325c251bf38384dd Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Wed, 18 Mar 2026 15:52:17 -0700 Subject: [PATCH] integrate structology marshaller --- gateway/router/marshal/json/marshal.go | 5 +- gateway/router/route.go | 7 +- go.mod | 4 +- go.sum | 8 +- internal/translator/rule.go | 17 +- internal/translator/service.go | 8 +- repository/component.go | 12 +- repository/content/content.go | 96 ++++++- repository/content/json_marshaller.go | 298 +++++++++++++++++++++ repository/content/json_marshaller_test.go | 219 +++++++++++++++ service.go | 5 +- view/extension/init.go | 3 + view/state/kind/locator/body.go | 29 +- view/state/parameters.go | 21 +- 14 files changed, 687 insertions(+), 45 deletions(-) create mode 100644 repository/content/json_marshaller.go create mode 100644 repository/content/json_marshaller_test.go diff --git a/gateway/router/marshal/json/marshal.go b/gateway/router/marshal/json/marshal.go index c17ffa80..095884da 100644 --- a/gateway/router/marshal/json/marshal.go +++ b/gateway/router/marshal/json/marshal.go @@ -2,12 +2,13 @@ package json import ( "bytes" + "reflect" + "unsafe" + "github.com/francoispqt/gojay" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/tagly/format/text" "github.com/viant/xunsafe" - "reflect" - "unsafe" ) const null = `null` diff --git a/gateway/router/route.go b/gateway/router/route.go index d38b499a..96d5874c 100644 --- a/gateway/router/route.go +++ b/gateway/router/route.go @@ -92,7 +92,10 @@ func (r *Route) UnmarshalFunc(request *http.Request) shared.Unmarshal { } return func(bytes []byte, i interface{}) error { - return r.Marshaller.JSON.JsonMarshaller.Unmarshal(bytes, i, jsonPathInterceptor, request) + if r.Marshaller.JSON.CanUnmarshal() { + return r.Marshaller.JSON.Unmarshal(bytes, i) + } + return r.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(bytes, i, jsonPathInterceptor, request) } } @@ -113,7 +116,7 @@ func (r *Route) Init(ctx context.Context, resource *Resource) error { return nil } r._unmarshallerInterceptors = r.Transforms.FilterByKind(marshal.TransformKindUnmarshal) - if err := r.Component.Content.InitMarshaller(r.Component.IOConfig(), r.Output.Exclude, r.BodyType(), r.OutputType()); err != nil { + if err := r.Component.Content.InitMarshaller(r.Component.IOConfig(), r.Output.Exclude, r.BodyType(), r.OutputType(), resource.Resource.LookupType()); err != nil { return err } if r.APIKey != nil { diff --git a/go.mod b/go.mod index c7c2ac2a..a49cdbde 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 github.com/viant/xreflect v0.7.3 - github.com/viant/xunsafe v0.10.3 + github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 google.golang.org/api v0.201.0 @@ -51,7 +51,7 @@ require ( github.com/viant/jsonrpc v0.17.0 github.com/viant/mcp v0.11.0 github.com/viant/mcp-protocol v0.11.0 - github.com/viant/structology v0.8.0 + github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 diff --git a/go.sum b/go.sum index a71dbbbb..28c2446c 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2p github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= -github.com/viant/structology v0.8.0 h1:WKdK67l+O1eqsubn8PWMhWcgspUGJ22SgJxUMfiRgqE= -github.com/viant/structology v0.8.0/go.mod h1:Fnm1DyR4gfyPbnhBMkQB5lR6/isYDnncBFO1nCxxmqs= +github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 h1:LUhy4A9ps2aFP3cBCTw4sMG1QmKWMNG1//vId03utEc= +github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= @@ -1228,8 +1228,8 @@ github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= github.com/viant/xreflect v0.7.3 h1:Oi2ZzSYWvs3lFBNMHEwHm6pu+3sQRlklzllycZyOGHk= github.com/viant/xreflect v0.7.3/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= -github.com/viant/xunsafe v0.10.3 h1:Fi4N+b5PH7e2iwT1UquAe7wUlTn4Fnb2kBnFLBixX+M= -github.com/viant/xunsafe v0.10.3/go.mod h1:V3RCwtqpbNPznhmHysyAOpsyuSVkIYWo1Ewip7qb9/s= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= diff --git a/internal/translator/rule.go b/internal/translator/rule.go index 6cea3bbd..e4c6f10d 100644 --- a/internal/translator/rule.go +++ b/internal/translator/rule.go @@ -3,6 +3,11 @@ package translator import ( "context" "fmt" + "os" + "path" + "path/filepath" + "strings" + "github.com/viant/afs" "github.com/viant/afs/url" "github.com/viant/datly/gateway/router" @@ -19,10 +24,6 @@ import ( "github.com/viant/datly/shared" "github.com/viant/datly/view" "github.com/viant/datly/view/state" - "os" - "path" - "path/filepath" - "strings" ) type ( @@ -325,13 +326,15 @@ func (r *Rule) applyDefaults() { setter.SetCaseFormatIfEmpty(&r.Route.Output.CaseFormat, "lc") setter.SetBoolIfFalse(&r.Input.IgnoreEmptyQueryParameters, r.IgnoreEmptyQueryParameters) setter.SetBoolIfFalse(&r.Input.CustomValidation, r.CustomValidation || r.Type != "") + setter.SetStringIfEmpty(&r.Route.Content.Marshaller.JSON.Engine, content.DefaultJSONEngineTypeName) if r.XMLUnmarshalType != "" { r.Route.Content.Marshaller.XML.TypeName = r.XMLUnmarshalType } if r.JSONMarshalType != "" { - r.Route.Content.Marshaller.JSON.TypeName = r.JSONMarshalType - } else if r.JSONUnmarshalType != "" { - r.Route.Content.Marshaller.JSON.TypeName = r.JSONUnmarshalType + r.Route.Content.Marshaller.JSON.MarshalTypeName = r.JSONMarshalType + } + if r.JSONUnmarshalType != "" { + r.Route.Content.Marshaller.JSON.UnmarshalTypeName = r.JSONUnmarshalType } } diff --git a/internal/translator/service.go b/internal/translator/service.go index 7367f6e3..37cc4f36 100644 --- a/internal/translator/service.go +++ b/internal/translator/service.go @@ -377,11 +377,11 @@ func (s *Service) persistRouterRule(ctx context.Context, resource *Resource, ser if resource.Rule.XMLUnmarshalType != "" { route.Content.Marshaller.XML.TypeName = resource.Rule.XMLUnmarshalType } - // JSON marshaller/unmarshaller customization: prefer MarshalType if provided, fallback to UnmarshalType. if resource.Rule.JSONMarshalType != "" { - route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONMarshalType - } else if resource.Rule.JSONUnmarshalType != "" { - route.Content.Marshaller.JSON.TypeName = resource.Rule.JSONUnmarshalType + route.Content.Marshaller.JSON.MarshalTypeName = resource.Rule.JSONMarshalType + } + if resource.Rule.JSONUnmarshalType != "" { + route.Content.Marshaller.JSON.UnmarshalTypeName = resource.Rule.JSONUnmarshalType } route.Component.Output.DataFormat = resource.Rule.DataFormat diff --git a/repository/component.go b/repository/component.go index 1c2c17bb..b78db0e8 100644 --- a/repository/component.go +++ b/repository/component.go @@ -121,11 +121,8 @@ func (c *Component) Init(ctx context.Context, resource *view.Resource) (err erro if err := c.initTransforms(ctx); err != nil { return nil } - if err := c.Content.InitMarshaller(c.IOConfig(), c.Output.Exclude, c.BodyType(), c.OutputType()); err != nil { - return err - } lookupType := resource.LookupType() - if err := c.Content.Marshaller.Init(lookupType); err != nil { + if err := c.Content.InitMarshaller(c.IOConfig(), c.Output.Exclude, c.BodyType(), c.OutputType(), lookupType); err != nil { return err } if err = c.Async.Init(ctx, resource, c.View); err != nil { @@ -450,10 +447,13 @@ func (c *Component) UnmarshalFor(opts ...UnmarshalOption) shared.Unmarshal { } } return func(data []byte, dest interface{}) error { + if c.Content.Marshaller.JSON.CanUnmarshal() { + return c.Content.Marshaller.JSON.Unmarshal(data, dest) + } if len(interceptors) > 0 || req != nil { - return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest, interceptors, req) + return c.Content.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(data, dest, interceptors, req) } - return c.Content.Marshaller.JSON.JsonMarshaller.Unmarshal(data, dest) + return c.Content.Marshaller.JSON.RuntimeUnmarshallerEngine().Unmarshal(data, dest) } } diff --git a/repository/content/content.go b/repository/content/content.go index a479fbaa..9183afce 100644 --- a/repository/content/content.go +++ b/repository/content/content.go @@ -2,6 +2,9 @@ package content import ( "fmt" + "reflect" + "strings" + "github.com/viant/datly/gateway/router/marshal" "github.com/viant/datly/gateway/router/marshal/config" "github.com/viant/datly/gateway/router/marshal/json" @@ -12,8 +15,6 @@ import ( "github.com/viant/xlsy" "github.com/viant/xmlify" "github.com/viant/xreflect" - "reflect" - "strings" ) const ( @@ -59,8 +60,14 @@ type ( } JSON struct { - Codec - JsonMarshaller *json.Marshaller + Engine string `json:",omitempty" yaml:",omitempty"` + MarshalTypeName string `json:",omitempty" yaml:",omitempty"` + UnmarshalTypeName string `json:",omitempty" yaml:",omitempty"` + marshalCodec Codec `json:"-" yaml:"-"` + unmarshalCodec Codec `json:"-" yaml:"-"` + JsonMarshaller *json.Marshaller `json:"-" yaml:"-"` + RuntimeMarshaller JSONMarshallerEngine `json:"-" yaml:"-"` + RuntimeUnmarshaller JSONUnmarshallerEngine `json:"-" yaml:"-"` } XLS struct { @@ -86,6 +93,14 @@ type ( Marshaller interface { Marshal(src interface{}) ([]byte, error) } + + JSONMarshallerEngine interface { + Marshal(src interface{}, options ...interface{}) ([]byte, error) + } + + JSONUnmarshallerEngine interface { + Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error + } ) func (u *Codec) CanUnmarshal() bool { @@ -117,7 +132,7 @@ func (m *Marshallers) Init(lookupType xreflect.LookupType) error { if err := m.JSON.Init(lookupType); err != nil { return err } - if err := m.XML.Init(lookupType); err != nil { + if err := m.XML.init(lookupType, false, true); err != nil { return err } if err := m.CSV.Init(lookupType); err != nil { @@ -127,6 +142,10 @@ func (m *Marshallers) Init(lookupType xreflect.LookupType) error { } func (u *Codec) Init(lookupType xreflect.LookupType) error { + return u.init(lookupType, true, true) +} + +func (u *Codec) init(lookupType xreflect.LookupType, requireMarshal, requireUnmarshal bool) error { if u.TypeName == "" { return nil } @@ -144,12 +163,57 @@ func (u *Codec) Init(lookupType xreflect.LookupType) error { if ok { u.marshal = marshaller.Marshal } - if u.marshal == nil && u.unmarshal == nil { - return fmt.Errorf("invalid type %s: unmarshaller/marshaller were not initialized", u.TypeName) + if requireMarshal && u.marshal == nil { + return fmt.Errorf("invalid type %s: marshaller was not initialized", u.TypeName) + } + if requireUnmarshal && u.unmarshal == nil { + return fmt.Errorf("invalid type %s: unmarshaller was not initialized", u.TypeName) + } + return nil +} + +func (j *JSON) Init(lookupType xreflect.LookupType) error { + j.marshalCodec = Codec{TypeName: j.MarshalTypeName} + j.unmarshalCodec = Codec{TypeName: j.UnmarshalTypeName} + if err := j.marshalCodec.init(lookupType, true, false); err != nil { + return err + } + if err := j.unmarshalCodec.init(lookupType, false, true); err != nil { + return err } return nil } +func (j *JSON) CanMarshal() bool { + return j.marshalCodec.CanMarshal() +} + +func (j *JSON) CanUnmarshal() bool { + return j.unmarshalCodec.CanUnmarshal() +} + +func (j *JSON) Marshal(src interface{}) ([]byte, error) { + return j.marshalCodec.Marshal(src) +} + +func (j *JSON) Unmarshal(bytes []byte, dest interface{}) error { + return j.unmarshalCodec.Unmarshal(bytes, dest) +} + +func (j *JSON) RuntimeMarshallerEngine() JSONMarshallerEngine { + if j.RuntimeMarshaller != nil { + return j.RuntimeMarshaller + } + return j.JsonMarshaller +} + +func (j *JSON) RuntimeUnmarshallerEngine() JSONUnmarshallerEngine { + if j.RuntimeUnmarshaller != nil { + return j.RuntimeUnmarshaller + } + return j.JsonMarshaller +} + func (c *Content) UnmarshallerInterceptors() marshal.Transforms { return c.unmarshallerInterceptors } @@ -176,9 +240,19 @@ func (x *XLSConfig) Options() []xlsy.Option { return options } -func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inputType, outputType reflect.Type) error { +func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inputType, outputType reflect.Type, lookupType xreflect.LookupType) error { c.unmarshallerInterceptors = c.Transforms.FilterByKind(marshal.TransformKindUnmarshal) - c.Marshaller.JSON.JsonMarshaller = json.New(config) + if err := c.Marshaller.Init(lookupType); err != nil { + return err + } + legacyMarshaller := json.New(config) + c.Marshaller.JSON.JsonMarshaller = legacyMarshaller + runtimeMarshaller, runtimeUnmarshaller, err := newJSONMarshaller(config, c.Marshaller.JSON.Engine, legacyMarshaller, lookupType) + if err != nil { + return err + } + c.Marshaller.JSON.RuntimeMarshaller = runtimeMarshaller + c.Marshaller.JSON.RuntimeUnmarshaller = runtimeUnmarshaller c.Marshaller.XLS.XlsMarshaller = xlsy.NewMarshaller(c.XLS.Options()...) if err := c.initCSVIfNeeded(inputType, outputType); err != nil { @@ -439,14 +513,14 @@ func (c *Content) Marshal(format string, field string, response interface{}, opt if field != "" { responseData := ensureSliceValue(response) tabJSONInterceptors := c.tabJSONInterceptors(field, responseData) - return c.Marshaller.JSON.JsonMarshaller.Marshal(response, tabJSONInterceptors) + return c.Marshaller.JSON.RuntimeMarshallerEngine().Marshal(response, tabJSONInterceptors) } return c.TabularJSON.OutputMarshaller.Marshal(response, options...) case JSONFormat: if c.Marshaller.JSON.CanMarshal() { return c.Marshaller.JSON.Marshal(response) } - return c.Marshaller.JSON.JsonMarshaller.Marshal(response, options...) + return c.Marshaller.JSON.RuntimeMarshallerEngine().Marshal(response, options...) default: return nil, fmt.Errorf("unsupproted readerData format: %s", format) } diff --git a/repository/content/json_marshaller.go b/repository/content/json_marshaller.go new file mode 100644 index 00000000..a360997e --- /dev/null +++ b/repository/content/json_marshaller.go @@ -0,0 +1,298 @@ +package content + +import ( + "fmt" + "net/http" + "reflect" + "strings" + + "github.com/viant/datly/gateway/router/marshal/config" + legacyjson "github.com/viant/datly/gateway/router/marshal/json" + structjson "github.com/viant/structology/encoding/json" + "github.com/viant/tagly/format" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" + "github.com/viant/xunsafe" +) + +var DefaultJSONEngineTypeName = reflect.TypeOf(StructologyJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(StructologyJSONRuntime{}).Name() + +func newJSONMarshaller(ioConfig *config.IOConfig, engine string, legacy *legacyjson.Marshaller, lookupType xreflect.LookupType) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + typeName := normalizeJSONEngineTypeName(engine) + if rType := xunsafe.LookupType(typeName); rType != nil { + return newJSONMarshallerByReflectType(rType, typeName, ioConfig, legacy) + } + + if lookupType == nil { + return nil, nil, fmt.Errorf("unsupported json marshaller engine: %s", typeName) + } + return newJSONMarshallerByType(lookupType, typeName, ioConfig, legacy) +} + +func normalizeJSONEngineTypeName(engine string) string { + normalized := strings.TrimSpace(engine) + if normalized == "" { + return DefaultJSONEngineTypeName + } + return normalized +} + +func newJSONMarshallerByType(lookupType xreflect.LookupType, typeName string, ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + rType, err := lookupType(typeName) + if err != nil { + return nil, nil, err + } + return newJSONMarshallerByReflectType(rType, typeName, ioConfig, legacy) +} + +func newJSONMarshallerByReflectType(rType reflect.Type, typeName string, ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) (JSONMarshallerEngine, JSONUnmarshallerEngine, error) { + value := reflect.New(rType).Interface() + if initializer, ok := value.(JSONRuntimeInitializer); ok { + if err := initializer.InitJSONRuntime(ioConfig, legacy); err != nil { + return nil, nil, err + } + } + marshaller, ok := value.(JSONMarshallerEngine) + if !ok { + if codec, ok := value.(Marshaller); ok { + marshaller = marshalCodecAdapter{Marshaller: codec} + } + } + unmarshaller, ok := value.(JSONUnmarshallerEngine) + if !ok { + if codec, ok := value.(Unmarshaller); ok { + unmarshaller = unmarshalCodecAdapter{Unmarshaller: codec} + } + } + if marshaller == nil { + return nil, nil, fmt.Errorf("invalid type %s: json marshaller engine was not initialized", typeName) + } + if unmarshaller == nil { + return nil, nil, fmt.Errorf("invalid type %s: json unmarshaller engine was not initialized", typeName) + } + return marshaller, unmarshaller, nil +} + +type marshalCodecAdapter struct { + Marshaller +} + +func (a marshalCodecAdapter) Marshal(src interface{}, _ ...interface{}) ([]byte, error) { + return a.Marshaller.Marshal(src) +} + +type unmarshalCodecAdapter struct { + Unmarshaller +} + +func (a unmarshalCodecAdapter) Unmarshal(bytes []byte, dest interface{}, _ ...interface{}) error { + return a.Unmarshaller.Unmarshal(bytes, dest) +} + +type JSONRuntimeInitializer interface { + InitJSONRuntime(ioConfig *config.IOConfig, legacy *legacyjson.Marshaller) error +} + +type StructologyJSONRuntime struct { + config *config.IOConfig +} + +func (m *StructologyJSONRuntime) InitJSONRuntime(ioConfig *config.IOConfig, _ *legacyjson.Marshaller) error { + m.config = ioConfig + return nil +} + +func (m *StructologyJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + structologyOptions, err := m.marshalOptions(options) + if err != nil { + return nil, err + } + return structjson.Marshal(src, structologyOptions...) +} + +func (m *StructologyJSONRuntime) Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error { + structologyOptions, err := m.unmarshalOptions(options) + if err != nil { + return err + } + return structjson.Unmarshal(bytes, dest, structologyOptions...) +} + +func (m *StructologyJSONRuntime) marshalOptions(options []interface{}) ([]structjson.Option, error) { + result := []structjson.Option{ + structjson.WithOmitEmpty(m.config != nil && m.config.OmitEmpty), + structjson.WithNilSlicePolicy(structjson.NilSliceAsEmptyArray), + } + if m.config != nil { + if caseFormat := m.config.CaseFormat; caseFormat.IsDefined() { + result = append(result, structjson.WithPathNameTransformer(datlyPathNameTransformer{caseFormat: caseFormat})) + } + if timeLayout := m.config.GetTimeLayout(); timeLayout != "" { + result = append(result, structjson.WithFormatTag(&format.Tag{TimeLayout: timeLayout})) + } + } + + var filters []*legacyjson.FilterEntry + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case []*legacyjson.FilterEntry: + filters = append(filters, actual...) + case legacyjson.MarshalerInterceptors: + if len(actual) > 0 { + return nil, fmt.Errorf("structology engine does not support legacy marshal interceptors") + } + case *legacyjson.MarshallSession: + return nil, fmt.Errorf("structology engine does not support legacy marshal sessions") + default: + return nil, fmt.Errorf("structology engine does not support marshal option %T", option) + } + } + + if excluder := newDatlyPathFieldExcluder(m.config, filters); excluder != nil { + result = append(result, structjson.WithPathFieldExcluder(excluder)) + } + return result, nil +} + +func (m *StructologyJSONRuntime) unmarshalOptions(options []interface{}) ([]structjson.Option, error) { + var result []structjson.Option + if m.config != nil { + if caseFormat := m.config.CaseFormat; caseFormat.IsDefined() { + result = append(result, structjson.WithCaseFormat(caseFormat)) + } + if timeLayout := m.config.GetTimeLayout(); timeLayout != "" { + result = append(result, structjson.WithFormatTag(&format.Tag{TimeLayout: timeLayout})) + } + } + + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case legacyjson.UnmarshalerInterceptors: + if len(actual) > 0 { + return nil, fmt.Errorf("structology engine does not support legacy unmarshal interceptors") + } + case *legacyjson.UnmarshalSession: + return nil, fmt.Errorf("structology engine does not support legacy unmarshal sessions") + case *http.Request: + continue + default: + return nil, fmt.Errorf("structology engine does not support unmarshal option %T", option) + } + } + + return result, nil +} + +type LegacyJSONRuntime struct { + marshaller *legacyjson.Marshaller +} + +func (m *LegacyJSONRuntime) InitJSONRuntime(_ *config.IOConfig, legacy *legacyjson.Marshaller) error { + m.marshaller = legacy + return nil +} + +func (m *LegacyJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + if m.marshaller == nil { + return nil, fmt.Errorf("legacy json runtime was not initialized") + } + return m.marshaller.Marshal(src, options...) +} + +func (m *LegacyJSONRuntime) Unmarshal(bytes []byte, dest interface{}, options ...interface{}) error { + if m.marshaller == nil { + return fmt.Errorf("legacy json runtime was not initialized") + } + return m.marshaller.Unmarshal(bytes, dest, options...) +} + +type datlyPathFieldExcluder struct { + exclude map[string]bool + filters map[string]map[string]bool +} + +type datlyPathNameTransformer struct { + caseFormat text.CaseFormat +} + +func newDatlyPathFieldExcluder(ioConfig *config.IOConfig, entries []*legacyjson.FilterEntry) structjson.PathFieldExcluder { + ret := &datlyPathFieldExcluder{} + if ioConfig != nil && len(ioConfig.Exclude) > 0 { + ret.exclude = ioConfig.Exclude + } + if len(entries) > 0 { + ret.filters = make(map[string]map[string]bool, len(entries)) + for _, entry := range entries { + if entry == nil { + continue + } + fields := make(map[string]bool, len(entry.Fields)) + for _, field := range entry.Fields { + fields[field] = true + } + ret.filters[entry.Path] = fields + normalizedPath := normalizeFilterPath(entry.Path) + if normalizedPath != entry.Path { + ret.filters[normalizedPath] = fields + } + } + } + if len(ret.exclude) == 0 && len(ret.filters) == 0 { + return nil + } + return ret +} + +func (d *datlyPathFieldExcluder) ExcludePath(path []string, fieldName string) bool { + fullPath := fieldName + parentPath := "" + if len(path) > 0 { + parentPath = strings.Join(path, ".") + fullPath = parentPath + "." + fieldName + } + if len(d.exclude) > 0 { + if d.exclude[fullPath] || d.exclude[config.NormalizeExclusionKey(fullPath)] { + return true + } + } + if len(d.filters) == 0 { + return false + } + fields, ok := d.filters[parentPath] + if !ok { + fields, ok = d.filters[normalizeFilterPath(parentPath)] + if !ok { + return false + } + } + return !fields[fieldName] +} + +func normalizeFilterPath(path string) string { + if path == "" { + return "" + } + return strings.ToLower(strings.ReplaceAll(path, "_", "")) +} + +func (d datlyPathNameTransformer) TransformPath(_ []string, fieldName string) string { + if fieldName == "ID" { + switch d.caseFormat { + case text.CaseFormatLower, text.CaseFormatLowerCamel, text.CaseFormatLowerUnderscore: + return "id" + case text.CaseFormatUpperCamel, text.CaseFormatUpper, text.CaseFormatUpperUnderscore: + return "ID" + } + } + fromCaseFormat := text.CaseFormatUpperCamel + if detected := text.DetectCaseFormat(fieldName); detected.IsDefined() { + fromCaseFormat = detected + } + return fromCaseFormat.Format(fieldName, d.caseFormat) +} diff --git a/repository/content/json_marshaller_test.go b/repository/content/json_marshaller_test.go new file mode 100644 index 00000000..81f0b4ac --- /dev/null +++ b/repository/content/json_marshaller_test.go @@ -0,0 +1,219 @@ +package content + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/francoispqt/gojay" + "github.com/stretchr/testify/require" + "github.com/viant/datly/gateway/router/marshal/config" + legacyjson "github.com/viant/datly/gateway/router/marshal/json" + "github.com/viant/tagly/format/text" + "github.com/viant/xreflect" +) + +type runtimeEngineCodec struct{} + +var legacyJSONEngineTypeName = reflect.TypeOf(LegacyJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(LegacyJSONRuntime{}).Name() + +func (r *runtimeEngineCodec) Marshal(src interface{}, _ ...interface{}) ([]byte, error) { + return []byte(`{"engine":"custom"}`), nil +} + +func (r *runtimeEngineCodec) Unmarshal(bytes []byte, dest interface{}, _ ...interface{}) error { + target := dest.(*map[string]interface{}) + *target = map[string]interface{}{"engine": "custom"} + return nil +} + +func TestNewJSONMarshaller_DefaultsToStructology(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyJSONEngineTypeName: + return reflect.TypeOf(LegacyJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, "", legacy, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*StructologyJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*StructologyJSONRuntime) + require.True(t, ok) +} + +func TestNewJSONMarshaller_UsesExplicitLegacyEngine(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyJSONEngineTypeName: + return reflect.TypeOf(LegacyJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, legacyJSONEngineTypeName, legacy, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*LegacyJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*LegacyJSONRuntime) + require.True(t, ok) +} + +func TestStructologyMarshaller_MarshalParity(t *testing.T) { + type eventType struct { + ID int + Type string + Extra string `json:"-"` + } + type payload struct { + UserID int + CreatedAt time.Time + Items []int + Name string + EventType eventType + Internal string `internal:"true"` + } + + cfg := &config.IOConfig{ + CaseFormat: text.CaseFormatLowerUnderscore, + TimeLayout: "2006-01-02", + } + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + marshaller, _, err := newJSONMarshaller(cfg, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + value := payload{ + UserID: 7, + CreatedAt: time.Date(2026, time.March, 5, 0, 0, 0, 0, time.UTC), + EventType: eventType{ID: 11, Type: "alpha", Extra: "ignored"}, + } + filters := []*legacyjson.FilterEntry{ + {Fields: []string{"UserID", "CreatedAt", "Items", "EventType"}}, + {Path: "EventType", Fields: []string{"Type"}}, + } + + actual, err := marshaller.Marshal(value, filters) + require.NoError(t, err) + + expected, err := legacy.Marshal(value, filters) + require.NoError(t, err) + + require.JSONEq(t, string(expected), string(actual)) + require.JSONEq(t, `{"user_id":7,"created_at":"2026-03-05","items":[],"event_type":{"type":"alpha"}}`, string(actual)) +} + +func TestStructologyMarshaller_UnmarshalUsesStructologyForBasicCase(t *testing.T) { + cfg := &config.IOConfig{CaseFormat: text.CaseFormatLowerCamel} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + _, unmarshaller, err := newJSONMarshaller(cfg, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + BuildTimeMs int `json:",omitempty"` + Changed bool + } + + var actual payload + err = unmarshaller.Unmarshal([]byte(`{"buildTimeMs":12,"changed":true}`), &actual) + + require.NoError(t, err) + require.Equal(t, 12, actual.BuildTimeMs) + require.True(t, actual.Changed) +} + +func TestStructologyMarshaller_MarshalRejectsLegacyInterceptors(t *testing.T) { + legacy := legacyjson.New(&config.IOConfig{}) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + marshaller, _, err := newJSONMarshaller(&config.IOConfig{}, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + Total int + } + + _, err = marshaller.Marshal(payload{Total: 3}, legacyjson.MarshalerInterceptors{ + "Total": func() ([]byte, error) { + return []byte(`3`), nil + }, + }) + + require.ErrorContains(t, err, "does not support legacy marshal interceptors") +} + +func TestStructologyMarshaller_UnmarshalRejectsLegacyInterceptors(t *testing.T) { + legacy := legacyjson.New(&config.IOConfig{}) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + _, unmarshaller, err := newJSONMarshaller(&config.IOConfig{}, DefaultJSONEngineTypeName, legacy, lookup) + require.NoError(t, err) + + type payload struct { + Total int + } + + var actual payload + err = unmarshaller.Unmarshal([]byte(`{"Total":3}`), &actual, legacyjson.UnmarshalerInterceptors{ + "Total": func(dst interface{}, decoder *gojay.Decoder, options ...interface{}) error { + return decoder.Int(dst.(*int)) + }, + }) + + require.ErrorContains(t, err, "does not support legacy unmarshal interceptors") +} + +func TestNewJSONMarshaller_UsesRegisteredEngineType(t *testing.T) { + cfg := &config.IOConfig{} + legacy := legacyjson.New(cfg) + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case "pkg.CustomJSONEngine": + return reflect.TypeOf(runtimeEngineCodec{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newJSONMarshaller(cfg, "pkg.CustomJSONEngine", legacy, lookup) + + require.NoError(t, err) + actual, err := marshaller.Marshal(struct{}{}) + require.NoError(t, err) + require.JSONEq(t, `{"engine":"custom"}`, string(actual)) + + var out map[string]interface{} + err = unmarshaller.Unmarshal([]byte(`{}`), &out) + require.NoError(t, err) + require.Equal(t, map[string]interface{}{"engine": "custom"}, out) +} diff --git a/service.go b/service.go index 2c984ddf..b48504b6 100644 --- a/service.go +++ b/service.go @@ -463,10 +463,7 @@ func (s *Service) ensureComponentInitialized(comp *repository.Component) error { return nil } // Initialize content marshallers as in Component.Init - if err := comp.Content.InitMarshaller(comp.IOConfig(), comp.Output.Exclude, comp.BodyType(), comp.OutputType()); err != nil { - return err - } - if err := comp.Content.Marshaller.Init(res.LookupType()); err != nil { + if err := comp.Content.InitMarshaller(comp.IOConfig(), comp.Output.Exclude, comp.BodyType(), comp.OutputType(), res.LookupType()); err != nil { return err } return nil diff --git a/view/extension/init.go b/view/extension/init.go index 7e0e679c..347898e6 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -6,6 +6,7 @@ import ( "mime/multipart" "net/http" + rcontent "github.com/viant/datly/repository/content" dcodec "github.com/viant/datly/view/extension/codec" "github.com/viant/datly/view/extension/handler" "github.com/viant/datly/view/extension/marshaller" @@ -81,6 +82,8 @@ func InitRegistry() { xreflect.NewType("auth.Token", xreflect.WithReflectType(reflect.TypeOf(&auth.Token{}))), xreflect.NewType("Token", xreflect.WithReflectType(reflect.TypeOf(&auth.Token{}))), xreflect.NewType("time.Location", xreflect.WithReflectType(reflect.TypeOf(&time.Location{}))), + xreflect.NewType("content.StructologyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyJSONRuntime{}))), + xreflect.NewType("content.LegacyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyJSONRuntime{}))), xreflect.NewType("marshaller.JSON", xreflect.WithReflectType(reflect.TypeOf(marshaller.JSON{}))), xreflect.NewType("marshaller.Gojay", xreflect.WithReflectType(reflect.TypeOf(marshaller.Gojay{}))), )), diff --git a/view/state/kind/locator/body.go b/view/state/kind/locator/body.go index e17af401..6374362a 100644 --- a/view/state/kind/locator/body.go +++ b/view/state/kind/locator/body.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "net/http" "reflect" + "strings" "sync" "github.com/viant/datly/shared" @@ -69,7 +70,7 @@ func (r *Body) Value(ctx context.Context, rType reflect.Type, name string) (inte if name == "" { return requestState.State(), true, nil } - sel, err := requestState.Selector(name) + sel, err := r.selectorByName(requestState, name) if err != nil { return nil, false, err } @@ -201,6 +202,32 @@ func (r *Body) ensureRequest(rType reflect.Type) (*structology.State, error) { return requestState, err } +func (r *Body) selectorByName(requestState *structology.State, name string) (*structology.Selector, error) { + sel, err := requestState.Selector(name) + if err == nil { + return sel, nil + } + stateType := requestState.Type() + for _, candidate := range stateType.RootSelectors() { + if jsonFieldName(candidate.Tag()) == name { + return candidate, nil + } + } + return nil, err +} + +func jsonFieldName(tag reflect.StructTag) string { + jsonTag := tag.Get("json") + if jsonTag == "" { + return "" + } + name := strings.SplitN(jsonTag, ",", 2)[0] + if name == "-" { + return "" + } + return name +} + func (r *Body) updateQueryString(ctx context.Context, body interface{}) { var queryParams map[string]string switch actual := body.(type) { diff --git a/view/state/parameters.go b/view/state/parameters.go index 24235aab..6964ea53 100644 --- a/view/state/parameters.go +++ b/view/state/parameters.go @@ -409,6 +409,10 @@ func (p *Parameter) buildField(pkgPath string, lookupType xreflect.LookupType) ( schema.rType = rType } fieldName := p.Name + tagFieldName := fieldName + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" { + tagFieldName = p.In.Name + } p.Schema.Cardinality = schema.Cardinality if p.Schema.Cardinality == Many && (rType.Kind() != reflect.Slice && rType.Kind() != reflect.Map) { rType = reflect.SliceOf(rType) @@ -417,11 +421,17 @@ func (p *Parameter) buildField(pkgPath string, lookupType xreflect.LookupType) ( if index := strings.LastIndex(fieldName, "."); index != -1 { fieldName = fieldName[index+1:] } + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" && fieldName == p.In.Name { + fieldName = SanitizeTypeName(fieldName) + } structField = reflect.StructField{Name: fieldName, Type: rType, PkgPath: xreflect.PkgPath(fieldName, pkgPath), - Tag: p.buildTag(fieldName), + Tag: p.buildTag(tagFieldName), + } + if p.In != nil && p.In.Kind == KindRequestBody && p.In.Name != "" && structField.Tag.Get("json") == "" { + structField.Tag = appendStructTag(structField.Tag, `json:"`+p.In.Name+`,omitempty"`) } if fieldName == rType.Name() && strings.Contains(p.Tag, "anonymous") { @@ -445,13 +455,20 @@ func buildMarkerFieldTag(structField reflect.StructField) stags.Tags { return updated } +func appendStructTag(tag reflect.StructTag, value string) reflect.StructTag { + if tag == "" { + return reflect.StructTag(value) + } + return reflect.StructTag(string(tag) + " " + value) +} + func (p Parameters) BuildBodyType(pkgPath string, lookupType xreflect.LookupType) (reflect.Type, error) { candidates := p.FilterByKind(KindRequestBody) bodyLeafParameters := make(Parameters, 0, len(candidates)) for i, candidate := range candidates { if candidate.In.Name != "" { bodyParameter := *candidates[i] - bodyParameter.Name = candidate.In.Name + bodyParameter.Name = SanitizeTypeName(candidate.In.Name) bodyLeafParameters = append(bodyLeafParameters, &bodyParameter) continue }