From b32be4ede550edaf0316fe2e6a45eb076506223d Mon Sep 17 00:00:00 2001 From: Terry Zhao Date: Tue, 24 Mar 2026 11:41:49 -0700 Subject: [PATCH] init --- go.mod | 5 +- go.sum | 11 +- repository/content/content.go | 51 +---- repository/content/tabjson_marshaller.go | 186 ++++++++++++++++++ repository/content/tabjson_marshaller_test.go | 102 ++++++++++ view/extension/init.go | 2 + 6 files changed, 306 insertions(+), 51 deletions(-) create mode 100644 repository/content/tabjson_marshaller.go create mode 100644 repository/content/tabjson_marshaller_test.go diff --git a/go.mod b/go.mod index d48aa85b..b7fe2218 100644 --- a/go.mod +++ b/go.mod @@ -32,9 +32,8 @@ require ( github.com/viant/structql v0.5.4 github.com/viant/toolbox v0.37.0 github.com/viant/velty v0.4.0 - github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e - + github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 golang.org/x/mod v0.28.0 golang.org/x/oauth2 v0.32.0 google.golang.org/api v0.201.0 @@ -52,7 +51,7 @@ require ( github.com/viant/jsonrpc v0.17.0 github.com/viant/mcp v0.11.0 github.com/viant/mcp-protocol v0.11.0 - github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 + github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 github.com/viant/tagly v0.3.0 github.com/viant/x v0.4.1-0.20260306005005-975ded1e1bef github.com/viant/xdatly v0.5.4-0.20260306062123-17850ac34977 diff --git a/go.sum b/go.sum index c82510f5..adf6a862 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588 h1:bnVgWzZzuz2p github.com/viant/sqlparser v0.11.1-0.20260224194657-0470849e3588/go.mod h1:2QRGiGZYk2/pjhORGG1zLVQ9JO+bXFhqIVi31mkCRPg= github.com/viant/sqlx v0.21.0 h1:Lx5KXmzfSjSvZZX5P0Ua9kFGvAmCxAjLOPe9pQA7VmY= github.com/viant/sqlx v0.21.0/go.mod h1:woTOwNiqvt6SqkI+5nyzlixcRTTV0IvLZUTberqb8mo= -github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76 h1:LUhy4A9ps2aFP3cBCTw4sMG1QmKWMNG1//vId03utEc= -github.com/viant/structology v0.8.1-0.20260318224343-dcb808a9bd76/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= +github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800 h1:NKLdUFp3tJsRBZrPhajSb1JqPRYWfTuquZBdy8XEOOo= +github.com/viant/structology v0.8.1-0.20260324183544-a0a56cb4c800/go.mod h1:AAFeViwniqua61sTKdOz/zlbLpN5vE4OVhDoiZJaMgA= github.com/viant/structql v0.5.4 h1:bMdcOpzU8UMoe5OBcyJVRxLAndvU1oj3ysvPUgBckCI= github.com/viant/structql v0.5.4/go.mod h1:nm9AYnAuSKH7b7pG+dKVxbQrr1Mgp1CQEMvUwwkE+I8= github.com/viant/tagly v0.3.0 h1:Y8IckveeSrroR8yisq4MBdxhcNqf4v8II01uCpamh4E= @@ -1226,13 +1226,10 @@ github.com/viant/xlsy v0.3.1 h1:KwA7PxOTVg+ns4CCPOdfNy5aEA9OUlIByUbuNC9ju0s= github.com/viant/xlsy v0.3.1/go.mod h1:RajfF9HkL/PfIxRCvZSubpNlpdMUNDKYZp8C1o3vF4Q= github.com/viant/xmlify v0.1.1 h1:Kmn7wnsq5APD8uJVP+kM6lIEGhSyjWSNOy4BvyfZQno= github.com/viant/xmlify v0.1.1/go.mod h1:w25+umH6nthlQ8ACT3K2/YJOLlbTXKLQXkdqFs6ky9s= - -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= -github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= - github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e h1:z4uCWPkSCnGwqbIc3ENoYJnYwtR2j/9eI79vO4vK9rQ= github.com/viant/xreflect v0.7.5-0.20260314170600-13f09f37d46e/go.mod h1:BwI+lqFjhKv2Vn4E0Jt6nvbwcFOWrM6H+sOMOX3JiU4= - +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559 h1:tQOsy7ov3XcTj+OXNF1apq9EKxSj82f5AjJCuhfCkMo= +github.com/viant/xunsafe v0.10.4-0.20260223225257-275a15956559/go.mod h1:RLSFNYewiF4p7+Lc18N4Zv4DHPWMTEky2VCLWvBdC5o= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg= github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= diff --git a/repository/content/content.go b/repository/content/content.go index 9183afce..c5302298 100644 --- a/repository/content/content.go +++ b/repository/content/content.go @@ -40,9 +40,10 @@ type ( TabularJSONConfig struct { FloatPrecision string + Engine string `json:",omitempty" yaml:",omitempty"` _config *tabjson.Config - InputMarhsaller *tabjson.Marshaller - OutputMarshaller *tabjson.Marshaller + InputMarhsaller TabularJSONUnmarshallerEngine `json:"-" yaml:"-"` + OutputMarshaller TabularJSONMarshallerEngine `json:"-" yaml:"-"` } XMLConfig struct { @@ -258,7 +259,7 @@ func (c *Content) InitMarshaller(config *config.IOConfig, exclude []string, inpu if err := c.initCSVIfNeeded(inputType, outputType); err != nil { return err } - if err := c.initTabJSONIfNeeded(exclude, inputType, outputType); err != nil { + if err := c.initTabJSONIfNeeded(exclude, inputType, outputType, lookupType); err != nil { return err } if err := c.initXMLIfNeeded(exclude, inputType, outputType); err != nil { @@ -307,50 +308,18 @@ func (c *Content) ensureCSV() { c.CSV = &CSVConfig{Separator: ","} } -func (c *Content) initTabJSONIfNeeded(excludedPaths []string, inputType reflect.Type, outputType reflect.Type) error { - - if c.TabularJSON == nil { - c.TabularJSON = &TabularJSONConfig{} - } - - if c.TabularJSON._config == nil { - c.TabularJSON._config = &tabjson.Config{} - } - - if c.TabularJSON._config.FieldSeparator == "" { - c.TabularJSON._config.FieldSeparator = "," - } - +func (c *Content) initTabJSONIfNeeded(excludedPaths []string, inputType reflect.Type, outputType reflect.Type, lookupType xreflect.LookupType) error { + c.TabularJSON = ensureTabularJSONConfig(c.TabularJSON, excludedPaths) if len(c.TabularJSON._config.FieldSeparator) != 1 { return fmt.Errorf("separator has to be a single char, but was %v", c.TabularJSON._config.FieldSeparator) } - - if c.TabularJSON._config.NullValue == "" { - c.TabularJSON._config.NullValue = "null" - } - - if c.TabularJSON.FloatPrecision != "" { - c.TabularJSON._config.StringifierConfig.StringifierFloat32Config.Precision = c.TabularJSON.FloatPrecision - c.TabularJSON._config.StringifierConfig.StringifierFloat64Config.Precision = c.TabularJSON.FloatPrecision - } - - c.TabularJSON._config.ExcludedPaths = excludedPaths - - if outputType.Kind() == reflect.Ptr { - outputType = outputType.Elem() - } - - var err error - c.TabularJSON.OutputMarshaller, err = tabjson.NewMarshaller(outputType, c.TabularJSON._config) + outputMarshaller, inputMarshaller, err := newTabularJSONMarshaller(c.TabularJSON, inputType, outputType, excludedPaths, lookupType) if err != nil { return err } - - if outputType == nil { - return nil - } - c.TabularJSON.InputMarhsaller, err = tabjson.NewMarshaller(inputType, nil) - return err + c.TabularJSON.OutputMarshaller = outputMarshaller + c.TabularJSON.InputMarhsaller = inputMarshaller + return nil } // func (c *Content) initXMLIfNeeded(excludedPaths []string, outputType reflect.Type, inputType reflect.Type) error { diff --git a/repository/content/tabjson_marshaller.go b/repository/content/tabjson_marshaller.go new file mode 100644 index 00000000..a6a02956 --- /dev/null +++ b/repository/content/tabjson_marshaller.go @@ -0,0 +1,186 @@ +package content + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/viant/datly/gateway/router/marshal/tabjson" + structjsontab "github.com/viant/structology/encoding/jsontab" + "github.com/viant/xreflect" + "github.com/viant/xunsafe" +) + +var DefaultTabularJSONEngineTypeName = reflect.TypeOf(StructologyTabularJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(StructologyTabularJSONRuntime{}).Name() + +type TabularJSONMarshallerEngine interface { + Marshal(src interface{}, options ...interface{}) ([]byte, error) +} + +type TabularJSONUnmarshallerEngine interface { + Unmarshal(bytes []byte, dest interface{}) error +} + +type TabularJSONRuntimeInitializer interface { + InitTabularJSONRuntime(cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) error +} + +func newTabularJSONMarshaller(cfg *TabularJSONConfig, inputType, outputType reflect.Type, excludedPaths []string, lookupType xreflect.LookupType) (TabularJSONMarshallerEngine, TabularJSONUnmarshallerEngine, error) { + typeName := normalizeTabularJSONEngineTypeName(cfg.Engine) + if rType := xunsafe.LookupType(typeName); rType != nil { + return newTabularJSONMarshallerByReflectType(rType, typeName, cfg, excludedPaths, inputType, outputType) + } + if lookupType == nil { + return nil, nil, fmt.Errorf("unsupported tabular json marshaller engine: %s", typeName) + } + rType, err := lookupType(typeName) + if err != nil { + return nil, nil, err + } + return newTabularJSONMarshallerByReflectType(rType, typeName, cfg, excludedPaths, inputType, outputType) +} + +func normalizeTabularJSONEngineTypeName(engine string) string { + normalized := strings.TrimSpace(engine) + if normalized == "" { + return DefaultTabularJSONEngineTypeName + } + return normalized +} + +func newTabularJSONMarshallerByReflectType(rType reflect.Type, typeName string, cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) (TabularJSONMarshallerEngine, TabularJSONUnmarshallerEngine, error) { + value := reflect.New(rType).Interface() + if initializer, ok := value.(TabularJSONRuntimeInitializer); ok { + if err := initializer.InitTabularJSONRuntime(cfg, excludedPaths, inputType, outputType); err != nil { + return nil, nil, err + } + } + marshaller, ok := value.(TabularJSONMarshallerEngine) + if !ok { + return nil, nil, fmt.Errorf("invalid type %s: tabular json marshaller engine was not initialized", typeName) + } + unmarshaller, ok := value.(TabularJSONUnmarshallerEngine) + if !ok { + return nil, nil, fmt.Errorf("invalid type %s: tabular json unmarshaller engine was not initialized", typeName) + } + return marshaller, unmarshaller, nil +} + +type StructologyTabularJSONRuntime struct { + config *TabularJSONConfig +} + +func (m *StructologyTabularJSONRuntime) InitTabularJSONRuntime(cfg *TabularJSONConfig, _ []string, _ reflect.Type, _ reflect.Type) error { + m.config = cfg + return nil +} + +func (m *StructologyTabularJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + jsontabOptions, err := m.marshalOptions(options) + if err != nil { + return nil, err + } + return structjsontab.Marshal(src, jsontabOptions...) +} + +func (m *StructologyTabularJSONRuntime) Unmarshal(bytes []byte, dest interface{}) error { + return structjsontab.Unmarshal(bytes, dest, m.unmarshalOptions()...) +} + +func (m *StructologyTabularJSONRuntime) marshalOptions(options []interface{}) ([]structjsontab.Option, error) { + result := []structjsontab.Option{ + structjsontab.WithTagName(tabjson.TagName), + } + if m.config != nil && m.config.FloatPrecision != "" { + precision, err := strconv.Atoi(strings.TrimSpace(m.config.FloatPrecision)) + if err != nil { + return nil, fmt.Errorf("invalid tabular json float precision %q: %w", m.config.FloatPrecision, err) + } + result = append(result, structjsontab.WithFloatPrecision(precision)) + } + for _, option := range options { + if option == nil { + continue + } + switch actual := option.(type) { + case []*tabjson.Config: + if len(actual) > 0 { + return nil, fmt.Errorf("structology tabular json engine does not support legacy depth configs") + } + default: + return nil, fmt.Errorf("structology tabular json engine does not support marshal option %T", option) + } + } + return result, nil +} + +func (m *StructologyTabularJSONRuntime) unmarshalOptions() []structjsontab.Option { + return []structjsontab.Option{ + structjsontab.WithTagName(tabjson.TagName), + } +} + +type LegacyTabularJSONRuntime struct { + input *tabjson.Marshaller + output *tabjson.Marshaller +} + +func (m *LegacyTabularJSONRuntime) InitTabularJSONRuntime(cfg *TabularJSONConfig, excludedPaths []string, inputType, outputType reflect.Type) error { + configured := ensureTabularJSONConfig(cfg, excludedPaths) + var err error + m.output, err = newLegacyTabularOutputMarshaller(outputType, configured._config) + if err != nil { + return err + } + if inputType == nil { + return nil + } + m.input, err = tabjson.NewMarshaller(inputType, nil) + return err +} + +func (m *LegacyTabularJSONRuntime) Marshal(src interface{}, options ...interface{}) ([]byte, error) { + if m.output == nil { + return nil, fmt.Errorf("legacy tabular json runtime was not initialized") + } + return m.output.Marshal(src, options...) +} + +func (m *LegacyTabularJSONRuntime) Unmarshal(bytes []byte, dest interface{}) error { + if m.input == nil { + return fmt.Errorf("legacy tabular json runtime was not initialized") + } + return m.input.Unmarshal(bytes, dest) +} + +func ensureTabularJSONConfig(cfg *TabularJSONConfig, excludedPaths []string) *TabularJSONConfig { + if cfg == nil { + cfg = &TabularJSONConfig{} + } + if cfg._config == nil { + cfg._config = &tabjson.Config{} + } + if cfg._config.FieldSeparator == "" { + cfg._config.FieldSeparator = "," + } + if cfg._config.NullValue == "" { + cfg._config.NullValue = "null" + } + if cfg.FloatPrecision != "" { + cfg._config.StringifierConfig.StringifierFloat32Config.Precision = cfg.FloatPrecision + cfg._config.StringifierConfig.StringifierFloat64Config.Precision = cfg.FloatPrecision + } + cfg._config.ExcludedPaths = excludedPaths + return cfg +} + +func newLegacyTabularOutputMarshaller(outputType reflect.Type, cfg *tabjson.Config) (*tabjson.Marshaller, error) { + if outputType == nil { + return nil, nil + } + if outputType.Kind() == reflect.Ptr { + outputType = outputType.Elem() + } + return tabjson.NewMarshaller(outputType, cfg) +} diff --git a/repository/content/tabjson_marshaller_test.go b/repository/content/tabjson_marshaller_test.go new file mode 100644 index 00000000..de349f46 --- /dev/null +++ b/repository/content/tabjson_marshaller_test.go @@ -0,0 +1,102 @@ +package content + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" + "github.com/viant/xreflect" +) + +var legacyTabularJSONEngineTypeName = reflect.TypeOf(LegacyTabularJSONRuntime{}).PkgPath() + "/" + reflect.TypeOf(LegacyTabularJSONRuntime{}).Name() + +func TestNewTabularJSONMarshaller_DefaultsToStructology(t *testing.T) { + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyTabularJSONEngineTypeName: + return reflect.TypeOf(LegacyTabularJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + marshaller, unmarshaller, err := newTabularJSONMarshaller(&TabularJSONConfig{}, reflect.TypeOf(struct{}{}), reflect.TypeOf(struct{}{}), nil, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*StructologyTabularJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*StructologyTabularJSONRuntime) + require.True(t, ok) +} + +func TestNewTabularJSONMarshaller_UsesExplicitLegacyEngine(t *testing.T) { + lookup := func(name string, _ ...xreflect.Option) (reflect.Type, error) { + switch name { + case legacyTabularJSONEngineTypeName: + return reflect.TypeOf(LegacyTabularJSONRuntime{}), nil + default: + return nil, fmt.Errorf("unknown type %s", name) + } + } + + cfg := &TabularJSONConfig{Engine: legacyTabularJSONEngineTypeName} + marshaller, unmarshaller, err := newTabularJSONMarshaller(cfg, reflect.TypeOf(struct{}{}), reflect.TypeOf(struct{}{}), nil, lookup) + + require.NoError(t, err) + _, ok := marshaller.(*LegacyTabularJSONRuntime) + require.True(t, ok) + _, ok = unmarshaller.(*LegacyTabularJSONRuntime) + require.True(t, ok) +} + +func TestStructologyTabularJSONRuntime_MarshalPrecisionAndNested(t *testing.T) { + type child struct { + ID int `csvName:"id"` + } + type row struct { + ID int `csvName:"id"` + Price float64 `csvName:"price"` + Children []child `csvName:"children"` + } + cfg := &TabularJSONConfig{FloatPrecision: "4"} + runtime := &StructologyTabularJSONRuntime{} + require.NoError(t, runtime.InitTabularJSONRuntime(cfg, nil, nil, nil)) + + actual, err := runtime.Marshal([]row{{ID: 1, Price: 1.123456, Children: []child{{ID: 10}, {ID: 11}}}}) + require.NoError(t, err) + require.JSONEq(t, `[["id","price","children"],[1,1.1235,[["id"],[10],[11]]]]`, string(actual)) +} + +func TestStructologyTabularJSONRuntime_Unmarshal(t *testing.T) { + type row struct { + ID int `csvName:"id"` + Name string `csvName:"name"` + } + runtime := &StructologyTabularJSONRuntime{} + require.NoError(t, runtime.InitTabularJSONRuntime(&TabularJSONConfig{}, nil, nil, nil)) + + var actual []row + err := runtime.Unmarshal([]byte(`[["id","name"],[1,"alpha"],[2,"beta"]]`), &actual) + require.NoError(t, err) + require.Equal(t, []row{{ID: 1, Name: "alpha"}, {ID: 2, Name: "beta"}}, actual) +} + +func TestContentInitTabJSONIfNeeded_DefaultStructology(t *testing.T) { + type row struct { + ID int `csvName:"id"` + } + content := &Content{} + + err := content.initTabJSONIfNeeded(nil, reflect.TypeOf([]row{}), reflect.TypeOf([]row{}), nil) + require.NoError(t, err) + + payload, err := content.TabularJSON.OutputMarshaller.Marshal([]row{{ID: 7}}) + require.NoError(t, err) + + var actual [][]interface{} + require.NoError(t, json.Unmarshal(payload, &actual)) + require.Equal(t, "id", actual[0][0]) + require.EqualValues(t, 7, actual[1][0]) +} diff --git a/view/extension/init.go b/view/extension/init.go index 347898e6..6b3b8abb 100644 --- a/view/extension/init.go +++ b/view/extension/init.go @@ -84,6 +84,8 @@ func InitRegistry() { xreflect.NewType("time.Location", xreflect.WithReflectType(reflect.TypeOf(&time.Location{}))), xreflect.NewType("content.StructologyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyJSONRuntime{}))), xreflect.NewType("content.LegacyJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyJSONRuntime{}))), + xreflect.NewType("content.StructologyTabularJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.StructologyTabularJSONRuntime{}))), + xreflect.NewType("content.LegacyTabularJSONRuntime", xreflect.WithReflectType(reflect.TypeOf(rcontent.LegacyTabularJSONRuntime{}))), xreflect.NewType("marshaller.JSON", xreflect.WithReflectType(reflect.TypeOf(marshaller.JSON{}))), xreflect.NewType("marshaller.Gojay", xreflect.WithReflectType(reflect.TypeOf(marshaller.Gojay{}))), )),