From 55b2e77c32cb7a4db6912841edaca8d4922aa2c6 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Tue, 4 Mar 2025 17:53:04 +0200 Subject: [PATCH 1/3] HCK-10162: install lodash --- package-lock.json | 12 ++++++++++-- package.json | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0456f4f..dad900d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,15 @@ { "name": "Teradata", - "version": "0.2.10", + "version": "0.2.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Teradata", - "version": "0.2.10", + "version": "0.2.12", + "dependencies": { + "lodash": "4.17.21" + }, "devDependencies": { "@hackolade/hck-esbuild-plugins-pack": "0.0.1", "@typescript-eslint/eslint-plugin": "7.11.0", @@ -2964,6 +2967,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", diff --git a/package.json b/package.json index 4fd4ae5..5eab22d 100644 --- a/package.json +++ b/package.json @@ -66,8 +66,8 @@ }, "devDependencies": { "@hackolade/hck-esbuild-plugins-pack": "0.0.1", - "@typescript-eslint/parser": "7.11.0", "@typescript-eslint/eslint-plugin": "7.11.0", + "@typescript-eslint/parser": "7.11.0", "esbuild": "0.20.2", "esbuild-plugin-clean": "1.0.1", "eslint": "8.57.0", @@ -79,5 +79,8 @@ "lint-staged": "14.0.1", "prettier": "3.2.5", "simple-git-hooks": "2.11.1" + }, + "dependencies": { + "lodash": "4.17.21" } -} \ No newline at end of file +} From 4d8ea207e79f6dfa09b36defd5b24190f874c84b Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Wed, 5 Mar 2025 09:52:07 +0200 Subject: [PATCH 2/3] HCK-10162: add dbt provider --- esbuild.package.js | 1 + forward_engineering/dbtProvider.js | 75 +++++++ forward_engineering/ddlProvider.js | 11 +- .../helpers/columnDefinitionHelper.js | 194 +++++++++--------- forward_engineering/helpers/keyHelper.js | 58 +++++- forward_engineering/types.d.ts | 45 ++++ 6 files changed, 279 insertions(+), 105 deletions(-) create mode 100644 forward_engineering/dbtProvider.js create mode 100644 forward_engineering/types.d.ts diff --git a/esbuild.package.js b/esbuild.package.js index 9b78a5c..8034c95 100644 --- a/esbuild.package.js +++ b/esbuild.package.js @@ -31,6 +31,7 @@ esbuild entryPoints: [ path.resolve(__dirname, 'forward_engineering', 'api.js'), path.resolve(__dirname, 'forward_engineering', 'ddlProvider.js'), + path.resolve(__dirname, 'forward_engineering', 'dbtProvider.js'), path.resolve(__dirname, 'reverse_engineering', 'api.js'), ], bundle: true, diff --git a/forward_engineering/dbtProvider.js b/forward_engineering/dbtProvider.js new file mode 100644 index 0000000..61f685b --- /dev/null +++ b/forward_engineering/dbtProvider.js @@ -0,0 +1,75 @@ +/** + * @typedef {import('./types').ColumnDefinition} ColumnDefinition + * @typedef {import('./types').ConstraintDto} ConstraintDto + * @typedef {import('./types').JsonSchema} JsonSchema + */ +const { identity, toLower, toUpper } = require('lodash'); + +const types = require('./configs/types'); +const defaultTypes = require('./configs/defaultTypes'); +const getKeyHelper = require('./helpers/keyHelper'); +const columnDefinitionHelper = require('./helpers/columnDefinitionHelper'); + +class DbtProvider { + /** + * @returns {DbtProvider} + */ + static createDbtProvider() { + return new DbtProvider(); + } + + /** + * @param {string} type + * @returns {string | undefined} + */ + getDefaultType(type) { + return defaultTypes[type]; + } + + /** + * @returns {Record} + */ + getTypesDescriptors() { + return types; + } + + /** + * @param {string} type + * @returns {boolean} + */ + hasType(type) { + return Object.keys(types).map(toLower).includes(toLower(type)); + } + + /** + * @param {{ columnDefinition: ColumnDefinition }} + * @returns {{ type: string; }} + */ + decorateType({ type, columnDefinition }) { + return columnDefinitionHelper.decorateType(toUpper(type), columnDefinition); + } + + /** + * @param {{ jsonSchema: JsonSchema }} + * @returns {ConstraintDto[]} + */ + getCompositeKeyConstraints({ jsonSchema }) { + const keyHelper = getKeyHelper(identity); + const compositePrimaryKeys = keyHelper.getCompositePrimaryKeys(jsonSchema); + const compositeUniqueKeys = keyHelper.getCompositeUniqueKeys(jsonSchema); + + return [...compositePrimaryKeys, ...compositeUniqueKeys]; + } + + /** + * @param {{ columnDefinition: ColumnDefinition; jsonSchema: JsonSchema }} + * @returns {ConstraintDto[]} + */ + getColumnConstraints({ columnDefinition, jsonSchema }) { + const keyHelper = getKeyHelper(identity); + + return keyHelper.getColumnConstraints({ columnDefinition, jsonSchema }); + } +} + +module.exports = DbtProvider; diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index bb25185..12c3e80 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -1,3 +1,4 @@ +const _ = require('lodash'); const templates = require('./configs/templates'); const defaultTypes = require('./configs/defaultTypes'); const types = require('./configs/types'); @@ -12,7 +13,6 @@ const { joinActivatedAndDeactivatedStatements } = require('./utils/joinActivated * @return {Object} */ module.exports = (baseProvider, options, app) => { - const _ = app.require('lodash'); const { assignTemplates } = app.require('@hackolade/ddl-fe-utils'); const { tab, @@ -32,7 +32,7 @@ module.exports = (baseProvider, options, app) => { divideIntoActivatedAndDeactivated, assignTemplates, }); - const keyHelper = require('./helpers/keyHelper')(_, clean); + const keyHelper = require('./helpers/keyHelper')(clean); const { getTableName, getIndexName, @@ -51,7 +51,7 @@ module.exports = (baseProvider, options, app) => { checkAllKeysDeactivated, divideIntoActivatedAndDeactivated, }); - const { decorateType } = require('./helpers/columnDefinitionHelper')(_); + const { decorateType } = require('./helpers/columnDefinitionHelper'); const additionalOptions = getAdditionalOptions(options.additionalOptions); @@ -196,10 +196,7 @@ module.exports = (baseProvider, options, app) => { name: databaseName, }); - return commentIfDeactivated( - [databaseStatement, createSessionStatement].join('\n'), - { isActivated }, - ); + return commentIfDeactivated([databaseStatement, createSessionStatement].join('\n'), { isActivated }); }, createUdt(udt, dbData) { diff --git a/forward_engineering/helpers/columnDefinitionHelper.js b/forward_engineering/helpers/columnDefinitionHelper.js index ecc66aa..0b8dd05 100644 --- a/forward_engineering/helpers/columnDefinitionHelper.js +++ b/forward_engineering/helpers/columnDefinitionHelper.js @@ -1,98 +1,98 @@ -module.exports = _ => { - const addLength = (type, length) => { - return `${type}(${length})`; - }; - - const addChildType = (type, childType) => addLength(type, childType); - - const addPrecision = (type, precision) => { - return `${type}(${precision})`; - }; - - const addScalePrecision = (type, precision, scale) => { - if (_.isNumber(scale)) { - return `${type}(${precision},${scale})`; - } else { - return addPrecision(type, precision); - } - }; - - const addTimezonePrecision = (type, precision, timezone) => { - const resultType = precision ? addPrecision(type, precision) : type; - if (timezone) { - return `${resultType} WITH TIME ZONE`; - } else { - return resultType; - } - }; - - const addPrecisionAndToPrecision = (type, precision, toPrecision, fractSecPrecision) => { - let typeStatement = type; - typeStatement += precision ? `(${precision})` : ''; - typeStatement += toPrecision ? ` TO ${toPrecision}` : ''; - typeStatement += fractSecPrecision ? `(${fractSecPrecision})` : ''; - - return typeStatement; - }; - - const canHaveLength = type => - [ - 'CHAR', - 'CHARACTER', - 'VARCHAR', - 'CHAR VARYING', - 'CHARACTER VARYING', - 'VARGRAPHIC', - 'CLOB', - 'CHARACTER LARGE OBJECT', - 'BYTE', - 'VARBYTE', - 'BLOB', - 'ARRAY', - 'JSON', - 'XML', - 'XMLTYPE', - 'ST_GEOMETRY', - ].includes(type); - - const canHavePrecisionAndScale = type => ['DECIMAL', 'DEC', 'NUMERIC', 'NUMBER'].includes(type); - - const canHavePrecisionAndToPrecision = type => - ['INTERVAL YEAR', 'INTERVAL DAY', 'INTERVAL HOUR', 'INTERVAL MINUTE'].includes(type); - - const canHavePrecisionAndSecondPrecision = type => ['INTERVAL SECOND'].includes(type); - - const canHaveFractionalSecondPrecision = type => ['TIME', 'TIMESTAMP'].includes(type); - - const canHaveChildType = type => ['PERIOD'].includes(type); - - const decorateType = (type, columnDefinition) => { - type = _.toUpper(type); - - if (canHaveChildType(type)) { - const childValueType = decorateType(columnDefinition.childValueType, columnDefinition); - return addChildType(type, childValueType); - } else if (canHaveLength(type) && _.isNumber(columnDefinition.length)) { - return addLength(type, columnDefinition.length); - } else if (canHavePrecisionAndToPrecision(type)) { - return addPrecisionAndToPrecision( - type, - columnDefinition.precision, - columnDefinition.toPrecision, - columnDefinition.fractSecPrecision, - ); - } else if (canHavePrecisionAndSecondPrecision(type)) { - return addScalePrecision(type, columnDefinition.precision, columnDefinition.fractSecPrecision); - } else if (canHavePrecisionAndScale(type) && _.isNumber(columnDefinition.precision)) { - return addScalePrecision(type, columnDefinition.precision, columnDefinition.scale); - } else if (canHaveFractionalSecondPrecision(type)) { - return addTimezonePrecision(type, columnDefinition.fractSecPrecision, columnDefinition.timezone); - } - - return type; - }; - - return { - decorateType, - }; +const _ = require('lodash'); + +const addLength = (type, length) => { + return `${type}(${length})`; +}; + +const addChildType = (type, childType) => addLength(type, childType); + +const addPrecision = (type, precision) => { + return `${type}(${precision})`; +}; + +const addScalePrecision = (type, precision, scale) => { + if (_.isNumber(scale)) { + return `${type}(${precision},${scale})`; + } else { + return addPrecision(type, precision); + } +}; + +const addTimezonePrecision = (type, precision, timezone) => { + const resultType = precision ? addPrecision(type, precision) : type; + if (timezone) { + return `${resultType} WITH TIME ZONE`; + } else { + return resultType; + } +}; + +const addPrecisionAndToPrecision = (type, precision, toPrecision, fractSecPrecision) => { + let typeStatement = type; + typeStatement += precision ? `(${precision})` : ''; + typeStatement += toPrecision ? ` TO ${toPrecision}` : ''; + typeStatement += fractSecPrecision ? `(${fractSecPrecision})` : ''; + + return typeStatement; +}; + +const canHaveLength = type => + [ + 'CHAR', + 'CHARACTER', + 'VARCHAR', + 'CHAR VARYING', + 'CHARACTER VARYING', + 'VARGRAPHIC', + 'CLOB', + 'CHARACTER LARGE OBJECT', + 'BYTE', + 'VARBYTE', + 'BLOB', + 'ARRAY', + 'JSON', + 'XML', + 'XMLTYPE', + 'ST_GEOMETRY', + ].includes(type); + +const canHavePrecisionAndScale = type => ['DECIMAL', 'DEC', 'NUMERIC', 'NUMBER'].includes(type); + +const canHavePrecisionAndToPrecision = type => + ['INTERVAL YEAR', 'INTERVAL DAY', 'INTERVAL HOUR', 'INTERVAL MINUTE'].includes(type); + +const canHavePrecisionAndSecondPrecision = type => ['INTERVAL SECOND'].includes(type); + +const canHaveFractionalSecondPrecision = type => ['TIME', 'TIMESTAMP'].includes(type); + +const canHaveChildType = type => ['PERIOD'].includes(type); + +const decorateType = (type, columnDefinition) => { + type = _.toUpper(type); + + if (canHaveChildType(type)) { + const childValueType = decorateType(columnDefinition.childValueType, columnDefinition); + return addChildType(type, childValueType); + } else if (canHaveLength(type) && _.isNumber(columnDefinition.length)) { + return addLength(type, columnDefinition.length); + } else if (canHavePrecisionAndToPrecision(type)) { + return addPrecisionAndToPrecision( + type, + columnDefinition.precision, + columnDefinition.toPrecision, + columnDefinition.fractSecPrecision, + ); + } else if (canHavePrecisionAndSecondPrecision(type)) { + return addScalePrecision(type, columnDefinition.precision, columnDefinition.fractSecPrecision); + } else if (canHavePrecisionAndScale(type) && _.isNumber(columnDefinition.precision)) { + return addScalePrecision(type, columnDefinition.precision, columnDefinition.scale); + } else if (canHaveFractionalSecondPrecision(type)) { + return addTimezonePrecision(type, columnDefinition.fractSecPrecision, columnDefinition.timezone); + } + + return type; +}; + +module.exports = { + decorateType, }; diff --git a/forward_engineering/helpers/keyHelper.js b/forward_engineering/helpers/keyHelper.js index 30f6992..6d543f9 100644 --- a/forward_engineering/helpers/keyHelper.js +++ b/forward_engineering/helpers/keyHelper.js @@ -1,4 +1,12 @@ -module.exports = (_, clean) => { +/** + * @typedef {import('forward_engineering/types').JsonSchema} JsonSchema + * @typedef {import('forward_engineering/types').ColumnDefinition} ColumnDefinition + * @typedef {import('forward_engineering/types').ConstraintDto} ConstraintDto + */ + +const _ = require('lodash'); + +module.exports = clean => { const mapProperties = (jsonSchema, iteratee) => { return Object.entries(jsonSchema.properties).map(iteratee); }; @@ -128,9 +136,57 @@ module.exports = (_, clean) => { ]; }; + /** + * @param {{ columnDefinition: ColumnDefinition }} + * @returns {ConstraintDto | undefined} + */ + const getPrimaryKeyConstraint = ({ columnDefinition, jsonSchema }) => { + if (!isPrimaryKey(columnDefinition)) { + return; + } + + return hydrateKeyConstraintOptions( + columnDefinition.primaryKeyConstraintName, + 'PRIMARY KEY', + columnDefinition.name, + columnDefinition.isActivated, + ); + }; + + /** + * @param {{ columnDefinition: ColumnDefinition }} + * @returns {ConstraintDto | undefined} + */ + const getUniqueKeyConstraint = ({ columnDefinition, jsonSchema }) => { + if (!isUniqueKey(columnDefinition)) { + return; + } + + return hydrateKeyConstraintOptions( + columnDefinition.uniqueKeyConstraintName, + 'UNIQUE', + columnDefinition.name, + columnDefinition.isActivated, + ); + }; + + /** + * @param {{ columnDefinition: ColumnDefinition; jsonSchema: JsonSchema }} + * @returns {ConstraintDto[]} + */ + const getColumnConstraints = ({ columnDefinition, jsonSchema }) => { + const primaryKeyConstraints = getPrimaryKeyConstraint({ columnDefinition, jsonSchema }); + const uniqueKeyConstraints = getUniqueKeyConstraint({ columnDefinition, jsonSchema }); + + return [primaryKeyConstraints, uniqueKeyConstraints].filter(Boolean); + }; + return { getTableKeyConstraints, isInlineUnique, isInlinePrimaryKey, + getColumnConstraints, + getCompositeUniqueKeys, + getCompositePrimaryKeys, }; }; diff --git a/forward_engineering/types.d.ts b/forward_engineering/types.d.ts new file mode 100644 index 0000000..3b7d0a7 --- /dev/null +++ b/forward_engineering/types.d.ts @@ -0,0 +1,45 @@ +export type Tag = { + id: string, + name: string, + allowedValues?: Array<{ id: string, value: string }>, + description?: string, + orReplace?: boolean, + ifNotExist?: boolean, +}; + +export type ObjectTag = { + id: string, + tagName?: string, + tagValue?: string, +}; + +export type ColumnDefinition = { + name: string; + type: string; + nullable: boolean; + isActivated: boolean; + isCaseSensitive?: boolean; + length?: number; + precision?: number; + primaryKey?: boolean; + primaryKeyConstraintName?: string; + scale?: number; + timePrecision?: number; + unique?: boolean; + uniqueKeyConstraintName?: string; +}; + +export type ConstraintDtoColumn = { + name: string; + isActivated: boolean; +}; + +export type KeyType = 'PRIMARY KEY' | 'UNIQUE'; + +export type ConstraintDto = { + keyType: KeyType; + name: string; + columns?: ConstraintDtoColumn[]; +}; + +export type JsonSchema = Record; From 9b72882fa8e2fe6004afd3db672043d24d19d677 Mon Sep 17 00:00:00 2001 From: Serhii Filonenko Date: Wed, 5 Mar 2025 09:53:58 +0200 Subject: [PATCH 3/3] HCK-10162: enable dbt feature --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5eab22d..1b9235e 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "jsonDocument": true, "jsonSchema": true, "excel": true, - "plugin": true + "plugin": true, + "dbt": true }, "enableReverseEngineering": true, "disableChoices": true,