From 6642a03f6c0fe194999cd49b34b28f653b242322 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Mon, 9 Mar 2026 11:37:04 -0500 Subject: [PATCH 01/42] ISSUE-2103 Handle trailing and trailing statement comments --- cli/api/BUILD | 18 ++++- cli/api/dbadapters/tasks.ts | 44 +++++++++-- cli/api/tasks_test.ts | 77 +++++++++++++++++++ tests/integration/bigquery.spec.ts | 4 +- ...xample_trailing_comment_in_operations.sqlx | 13 ++++ ...ing_statement_comment_with_operations.sqlx | 12 +++ 6 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 cli/api/tasks_test.ts create mode 100644 tests/integration/bigquery_project/definitions/example_trailing_comment_in_operations.sqlx create mode 100644 tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx diff --git a/cli/api/BUILD b/cli/api/BUILD index 94d7bcdf2..273a5877b 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -1,12 +1,16 @@ package(default_visibility = ["//visibility:public"]) load("//tools:ts_library.bzl", "ts_library") +load("//testing:index.bzl", "ts_test_suite") ts_library( name = "api", srcs = glob( ["**/*.ts"], - exclude = ["utils/**/*.*"], + exclude = [ + "utils/**/*.*", + "**/*_test.ts", + ], ), deps = [ "//cli/api/utils", @@ -42,3 +46,15 @@ ts_library( "@npm//tmp", ], ) + +ts_test_suite( + name = "tests", + srcs = ["tasks_test.ts"], + deps = [ + "//cli/api", + "//protos:ts", + "//testing", + "@npm//@types/chai", + "@npm//chai", + ], +) diff --git a/cli/api/dbadapters/tasks.ts b/cli/api/dbadapters/tasks.ts index 8991e5bed..60344fca6 100644 --- a/cli/api/dbadapters/tasks.ts +++ b/cli/api/dbadapters/tasks.ts @@ -1,16 +1,44 @@ import { dataform } from "df/protos/ts"; export function concatenateQueries(statements: string[], modifier?: (mod: string) => string) { - return statements + const processed = statements .filter(statement => !!statement) .map(statement => statement.trim()) - .map(statement => - statement.length > 0 && statement.charAt(statement.length - 1) === ";" - ? statement.substring(0, statement.length - 1) - : statement - ) - .map(statement => (!!modifier ? modifier(statement) : statement)) - .join("\n;\n"); + .filter(statement => statement.length > 0); + + return processed + .map((statement, index) => { + const isLast = index === processed.length - 1; + const commentIndex = statement.indexOf("--"); + + let code: string; + let comment: string; + + if (commentIndex === 0) { + code = ""; + comment = statement; + } else if (commentIndex > 0) { + code = statement.substring(0, commentIndex).trimEnd(); + comment = statement.substring(commentIndex); + } else { + code = statement; + comment = ""; + } + + if (modifier && code.length > 0) { + code = modifier(code); + } + + if (!isLast && code.length > 0 && !code.endsWith(";")) { + code = code + ";"; + } + + if (code.length > 0 && comment.length > 0) { + return code + " " + comment; + } + return code || comment; + }) + .join("\n"); } export class Tasks { diff --git a/cli/api/tasks_test.ts b/cli/api/tasks_test.ts new file mode 100644 index 000000000..02485ca96 --- /dev/null +++ b/cli/api/tasks_test.ts @@ -0,0 +1,77 @@ +import { expect } from "chai"; + +import { concatenateQueries } from "df/cli/api/dbadapters/tasks"; +import { suite, test } from "df/testing"; + +suite("concatenateQueries", () => { + test("adds semicolons to non-last statements and joins with newlines", () => { + expect(concatenateQueries(["SELECT 1", "SELECT 2"])).deep.equals( + "SELECT 1;\nSELECT 2" + ); + }); + + test("does not add semicolon to the last statement", () => { + expect(concatenateQueries(["SELECT 1", "SELECT 2", "SELECT 3"])).deep.equals( + "SELECT 1;\nSELECT 2;\nSELECT 3" + ); + }); + + test("preserves existing semicolons on non-last statements", () => { + expect(concatenateQueries(["SELECT 1;", "SELECT 2"])).deep.equals( + "SELECT 1;\nSELECT 2" + ); + }); + + test("preserves existing semicolons on last statement", () => { + expect(concatenateQueries(["SELECT 1;", "SELECT 2;"])).deep.equals( + "SELECT 1;\nSELECT 2;" + ); + }); + + test("adds semicolon before trailing comment if necessary, verifies fix for https://github.com/dataform-co/dataform/issues/1231", () => { + expect(concatenateQueries(["SELECT 1 -- blah", "drop table foo; -- baz", "SELECT 2"])).deep.equals( + "SELECT 1; -- blah\ndrop table foo; -- baz\nSELECT 2" + ); + }); + + test("preserves comment-only lines, verifies fix for https://github.com/dataform-co/dataform/issues/2103", () => { + expect(concatenateQueries(["SELECT 1;", "--drop table foo", "SELECT 2", "--baz"])).deep.equals( + "SELECT 1;\n--drop table foo\nSELECT 2;\n--baz" + ); + }); + + test("trims whitespace from statements", () => { + expect(concatenateQueries([" SELECT 1 ", " SELECT 2 ", " SELECT 3 -- blah"])).deep.equals( + "SELECT 1;\nSELECT 2;\nSELECT 3 -- blah" + ); + }); + + test("filters out empty and falsy statements", () => { + expect(concatenateQueries(["SELECT 1", "", "SELECT 2"])).deep.equals( + "SELECT 1;\nSELECT 2" + ); + expect(concatenateQueries(["SELECT 1", undefined, "SELECT 2"])).deep.equals( + "SELECT 1;\nSELECT 2" + ); + }); + + test("returns empty string for empty input", () => { + expect(concatenateQueries([])).deep.equals(""); + }); + + test("returns single statement without semicolon", () => { + expect(concatenateQueries(["SELECT 1"])).deep.equals("SELECT 1"); + }); + + test("applies modifier to code portion of each statement", () => { + expect( + concatenateQueries(["SELECT 1", "SELECT 2"], stmt => `(${stmt})`) + ).deep.equals("(SELECT 1);\n(SELECT 2)"); + }); + + test("applies modifier only to code portion, not comment", () => { + expect( + concatenateQueries(["SELECT 1 --comment", "SELECT 2"], stmt => `(${stmt})`) + ).deep.equals("(SELECT 1); --comment\n(SELECT 2)"); + }); +}); diff --git a/tests/integration/bigquery.spec.ts b/tests/integration/bigquery.spec.ts index 01b03ce23..88d4ded4c 100644 --- a/tests/integration/bigquery.spec.ts +++ b/tests/integration/bigquery.spec.ts @@ -384,7 +384,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) const bqadapter = new ExecutionSql({ warehouse: "bigquery" }, "1.4.8"); const refresh = bqadapter.publishTasks(table, { fullRefresh: true }, { fields: [] }).build(); - const splitRefresh = refresh[0].statement.split("\n;\n"); + const splitRefresh = refresh[0].statement.split("\n"); expect([...splitRefresh.slice(0, 2), ...splitRefresh.slice(-2)]).to.eql([ ...table.preOps, ...table.postOps @@ -394,7 +394,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) .publishTasks(table, { fullRefresh: false }, { fields: [] }) .build(); - const splitIncrement = increment[0].statement.split("\n;\n"); + const splitIncrement = increment[0].statement.split("\n"); expect([...splitIncrement.slice(0, 2), ...splitIncrement.slice(-2)]).to.eql([ ...table.preOps, ...table.postOps diff --git a/tests/integration/bigquery_project/definitions/example_trailing_comment_in_operations.sqlx b/tests/integration/bigquery_project/definitions/example_trailing_comment_in_operations.sqlx new file mode 100644 index 000000000..24f79f5f8 --- /dev/null +++ b/tests/integration/bigquery_project/definitions/example_trailing_comment_in_operations.sqlx @@ -0,0 +1,13 @@ +config { + "type": "table", + "description": "Verifies fix for https://github.com/dataform-co/dataform/issues/2103" +} + +SELECT 1 AS foo + +pre_operations { + + DECLARE bar DATE DEFAULT DATE_TRUNC(CURRENT_DATE(), WEEK(MONDAY)); + + -- DECLARE bar DATE DEFAULT DATE_TRUNC(DATE('2026-01-01'), WEEK(MONDAY)) +} \ No newline at end of file diff --git a/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx b/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx new file mode 100644 index 000000000..4cf7a6b65 --- /dev/null +++ b/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx @@ -0,0 +1,12 @@ +config { + type: "table", + schema: "test", + name: "testtab", + description: "Verifies fix for https://github.com/dataform-co/dataform/issues/1231" +} + +post_operations { + drop table ${self()} +} + +select 1 as col1 -- test comment \ No newline at end of file From 237a5ee4158a0991d4a6ab27d0a3973bcf3ab339 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 01:25:54 +0200 Subject: [PATCH 02/42] Bump vm2 from 3.10.2 to 3.11.3 (#2168) * Bump vm2 from 3.10.2 to 3.11.3 Bumps [vm2](https://github.com/patriksimek/vm2) from 3.10.2 to 3.11.3. - [Release notes](https://github.com/patriksimek/vm2/releases) - [Changelog](https://github.com/patriksimek/vm2/blob/main/CHANGELOG.md) - [Commits](https://github.com/patriksimek/vm2/compare/v3.10.2...v3.11.3) --- updated-dependencies: - dependency-name: vm2 dependency-version: 3.11.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Fix vm2 sandbox execution errors and CallSite path stripping The upgrade of vm2 to 3.11.3 tightened sandbox security, stripping file path information from V8 CallSite objects during Error.prepareStackTrace and breaking the resolution of symlinked modules. This caused critical test failures ("Unable to find valid caller file") during Dataform compilation. This commit resolves these issues by: * Resolving `projectDir` to its absolute realpath before sandbox compilation to circumvent symlink boundaries (especially important for bazel runfiles). * Injecting `global.__dataform_current_file` into the NodeVM compiler configuration with a try-finally block to track cross-boundary macro scopes. * Adding a fallback in `core/utils.ts:getCallerFile` to read the injected global file path when the V8 stack trace path is stripped or undefined. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Rafal Hawrylak --- cli/vm/compile.ts | 25 ++++++++- core/utils.ts | 10 ++-- core/workflow_settings.ts | 5 +- examples/examples_test.ts | 8 ++- package.json | 2 +- testing/run_core.ts | 24 +++++++-- yarn.lock | 107 +++++++++++++++----------------------- 7 files changed, 103 insertions(+), 78 deletions(-) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index f5e8cfc1e..7faefb467 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -8,6 +8,7 @@ import { encode64 } from "df/common/protos"; import { dataform } from "df/protos/ts"; export function compile(compileConfig: dataform.ICompileConfig) { + compileConfig.projectDir = fs.realpathSync(path.resolve(compileConfig.projectDir)); if ( !fs.existsSync( path.join(compileConfig.projectDir, "node_modules", "@dataform", "core", "bundle.js") @@ -49,7 +50,23 @@ export function compile(compileConfig: dataform.ICompileConfig) { path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName) }, sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml"], - compiler + // vm2 3.11.3 strips file paths from V8 CallSite objects inside the sandbox, + // which breaks getCallerFile() in @dataform/core. Wrap each compiled module so + // the current file path is exposed via a global, used as a fallback when the + // stack-trace path is unavailable. The try/finally restores the previous value + // to keep nested requires (macros) consistent. + compiler: (code, filePath) => { + const compiledCode = compiler(code, filePath); + return ` + var __old_file = global.__dataform_current_file; + global.__dataform_current_file = ${JSON.stringify(filePath)}; + try { + ${compiledCode} + } finally { + global.__dataform_current_file = __old_file; + } + `; + } }); const dataformCoreVersion: string = userCodeVm.run( @@ -61,7 +78,11 @@ export function compile(compileConfig: dataform.ICompileConfig) { } return userCodeVm.run( - `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")`, + ` + global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed:", e); } })(); + global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); + return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}") + `, vmIndexFileName ); } diff --git a/core/utils.ts b/core/utils.ts index 306487cbd..9068ad669 100644 --- a/core/utils.ts +++ b/core/utils.ts @@ -75,9 +75,13 @@ export function getCallerFile(rootDir: string) { break; } if (!lastfile) { - // This is likely caused by Session.compileError() being called inside Session.compile(). - // If so, explicitly pass the filename to Session.compileError(). - throw new Error("Unable to find valid caller file; please report this issue."); + if ((global as any).__dataform_current_file) { + lastfile = (global as any).__dataform_current_file; + } else { + // This is likely caused by Session.compileError() being called inside Session.compile(). + // If so, explicitly pass the filename to Session.compileError(). + throw new Error("Unable to find valid caller file; please report this issue."); + } } return Path.relativePath(lastfile, rootDir); } diff --git a/core/workflow_settings.ts b/core/workflow_settings.ts index 761bbce13..37fb54db8 100644 --- a/core/workflow_settings.ts +++ b/core/workflow_settings.ts @@ -10,9 +10,10 @@ declare var __non_webpack_require__: any; const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require; export function readWorkflowSettings(failIfMissing: boolean = true): dataform.ProjectConfig { - const workflowSettingsYaml = maybeRequire("workflow_settings.yaml"); + const globalAny = global as any; + const workflowSettingsYaml = globalAny.workflowSettingsYaml || maybeRequire("workflow_settings.yaml"); // `dataform.json` is deprecated; new versions of Dataform Core prefer `workflow_settings.yaml`. - const dataformJson = maybeRequire("dataform.json"); + const dataformJson = globalAny.dataformJson || maybeRequire("dataform.json"); if (workflowSettingsYaml && dataformJson) { throw Error( diff --git a/examples/examples_test.ts b/examples/examples_test.ts index 8eacdc16c..5465e8f84 100644 --- a/examples/examples_test.ts +++ b/examples/examples_test.ts @@ -11,7 +11,13 @@ suite("examples", { parallel: true }, () => { ["stackoverflow_reporter", "extreme_weather_programming"].forEach(exampleProject => { test(`${exampleProject} runs`, async () => { - const projectDir = `examples/${exampleProject}`; + // compile() calls realpath on projectDir, which would jump out of bazel's + // symlinked runfiles tree and leave the project without its sibling + // node_modules. Materialize a real copy (dereference symlinks) so the + // project and node_modules below resolve under a single real path. + const originalProjectDir = `examples/${exampleProject}`; + const projectDir = `examples/${exampleProject}_copy`; + fs.copySync(originalProjectDir, projectDir, { dereference: true }); fs.copySync( "examples/node_modules/@dataform/core", `${projectDir}/node_modules/@dataform/core` diff --git a/package.json b/package.json index ee46472b9..44b58b9a0 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "uglify-js": "^3.7.7", "untildify": "^4.0.0", "url": "^0.11.0", - "vm2": "3.10.2", + "vm2": "3.11.3", "vsce": "^1.79.5", "vscode-jsonrpc": "^5.0.1", "vscode-languageclient": "^6.1.3", diff --git a/testing/run_core.ts b/testing/run_core.ts index 9575fba3a..a5f23e424 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -54,11 +54,12 @@ export function coreExecutionRequestFromPath( projectDir: string, projectConfigOverride?: dataform.ProjectConfig ): dataform.CoreExecutionRequest { + const resolvedProjectDir = fs.realpathSync(path.resolve(projectDir)); return dataform.CoreExecutionRequest.create({ compile: { compileConfig: { - projectDir, - filePaths: walkDirectoryForFilenames(projectDir), + projectDir: resolvedProjectDir, + filePaths: walkDirectoryForFilenames(resolvedProjectDir), projectConfigOverride } } @@ -90,13 +91,28 @@ export function runMainInVm( path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName) }, sourceExtensions: SOURCE_EXTENSIONS, - compiler + compiler: (code, filePath) => { + const compiledCode = compiler(code, filePath); + return ` + var __old_file = global.__dataform_current_file; + global.__dataform_current_file = ${JSON.stringify(filePath)}; + try { + ${compiledCode} + } finally { + global.__dataform_current_file = __old_file; + } + `; + } }); const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest); const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); const encodedCoreExecutionResponse = nodeVm.run( - `return require("@dataform/core").main("${encodedCoreExecutionRequest}")`, + ` + global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed run_core:", e); } })(); + global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); + return require("@dataform/core").main("${encodedCoreExecutionRequest}") + `, vmIndexFileName ); return decode64(dataform.CoreExecutionResponse, encodedCoreExecutionResponse); diff --git a/yarn.lock b/yarn.lock index b6caee7ed..5b0a95dda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -95,21 +95,6 @@ stream-events "^1.0.5" teeny-request "^10.0.0" -"@google-cloud/common@^3.6.0": - version "3.6.0" - resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b" - integrity "sha1-wvbaX3knmkqax8cfwC1YK+q5jos= sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q==" - dependencies: - "@google-cloud/projectify" "^2.0.0" - "@google-cloud/promisify" "^2.0.0" - arrify "^2.0.1" - duplexify "^4.1.1" - ent "^2.2.0" - extend "^3.0.2" - google-auth-library "^7.0.2" - retry-request "^4.1.1" - teeny-request "^7.0.0" - "@google-cloud/common@^6.0.0": version "6.0.0" resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-6.0.0.tgz#776d4f747f0a3844f4ce13e06a2268743064b563" @@ -155,11 +140,6 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-4.0.0.tgz#a906e533ebdd0f754dca2509933334ce58b8c8b1" integrity sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g== -"@google-cloud/projectify@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-4.0.0.tgz#d600e0433daf51b88c1fa95ac7f02e38e80a07be" - integrity sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA== - "@google-cloud/promisify@^4.0.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-4.1.0.tgz#df8b060f0121c6462233f5420738dcda09c6df4a" @@ -479,16 +459,6 @@ resolved "https://registry.yarnpkg.com/@types/readline-sync/-/readline-sync-1.4.3.tgz#eac55a39d5a349912062c9e5216cd550c07fd9c8" integrity "sha1-6sVaOdWjSZEgYsnlIWzVUMB/2cg= sha512-YP9NVli96E+qQLAF2db+VjnAUEeZcFVg4YnMgr8kpDUFwQBnj31rPLOVHmazbKQhaIkJ9cMHsZhpKdzUeL0KTg==" -"@types/request@^2.48.12": - version "2.48.13" - resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.13.tgz#abdf4256524e801ea8fdda54320f083edb5a6b80" - integrity sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg== - dependencies: - "@types/caseless" "*" - "@types/node" "*" - "@types/tough-cookie" "*" - form-data "^2.5.5" - "@types/request@^2.48.3": version "2.48.4" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.4.tgz#df3d43d7b9ed3550feaa1286c6eabf0738e6cf7e" @@ -738,7 +708,7 @@ acorn@^7.4.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity "sha1-/q7SVZc9LndVW4PbwIhRpsY1IPo= sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==" -acorn@^8.11.0, acorn@^8.14.1, acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2: +acorn@^8.11.0, acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2: version "8.15.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816" integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg== @@ -1491,13 +1461,6 @@ domutils@^1.5.1: dom-serializer "0" domelementtype "1" -dot-prop@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.2.0.tgz#c34ecc29556dc45f1f4c22697b6f4904e0cc4fcb" - integrity "sha1-w07MKVVtxF8fTCJpe29JBODMT8s= sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==" - dependencies: - is-obj "^2.0.0" - dunder-proto@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" @@ -1517,16 +1480,6 @@ duplexify@^4.1.3: readable-stream "^3.1.1" stream-shift "^1.0.2" -duplexify@^4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.3.tgz#a07e1c0d0a2c001158563d32592ba58bddb0236f" - integrity sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA== - dependencies: - end-of-stream "^1.4.1" - inherits "^2.0.3" - readable-stream "^3.1.1" - stream-shift "^1.0.2" - ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -2091,11 +2044,6 @@ get-proto@^1.0.1: dunder-proto "^1.0.1" es-object-atoms "^1.0.0" -get-stream@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.0.tgz#3e0012cb6827319da2706e601a1583e8629a6718" - integrity "sha1-PgASy2gnMZ2icG5gGhWD6GKaZxg= sha512-A1B3Bh1UmL0bidM/YX2NsCOTnGJePL9rO/M+Mw3m9f2gUpfokS0hi5Eah0WSUEWZdZhIZtMjkIYS7mDfOqNHbg==" - get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" @@ -2177,6 +2125,23 @@ google-auth-library@^10.0.0-rc.1: gtoken "^8.0.0" jws "^4.0.0" +google-auth-library@^9.6.3: + version "9.15.1" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.15.1.tgz#0c5d84ed1890b2375f1cd74f03ac7b806b392928" + integrity sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng== + dependencies: + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + gaxios "^6.1.1" + gcp-metadata "^6.1.0" + gtoken "^7.0.0" + jws "^4.0.0" + +google-logging-utils@^0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-0.0.2.tgz#5fd837e06fa334da450433b9e3e1870c1594466a" + integrity sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ== + google-logging-utils@^1.0.0: version "1.1.3" resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-1.1.3.tgz#17b71f1f95d266d2ddd356b8f00178433f041b17" @@ -3965,11 +3930,6 @@ stream-shift@^1.0.2: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b" integrity sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ== -stream-shift@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b" - integrity sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ== - "string-width-cjs@npm:string-width@^4.2.0": version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" @@ -4002,8 +3962,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.1: - name strip-ansi-cjs +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -4017,6 +3976,13 @@ strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.2.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.2.0.tgz#d22a269522836a627af8d04b5c3fd2c7fa3e32e3" @@ -4095,6 +4061,17 @@ teeny-request@^10.0.0: node-fetch "^3.3.2" stream-events "^1.0.5" +teeny-request@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-9.0.0.tgz#18140de2eb6595771b1b02203312dfad79a4716d" + integrity sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g== + dependencies: + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.9" + stream-events "^1.0.5" + uuid "^9.0.0" + terser-webpack-plugin@^5.3.16: version "5.3.16" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz#741e448cc3f93d8026ebe4f7ef9e4afacfd56330" @@ -4460,12 +4437,12 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -vm2@3.10.2: - version "3.10.2" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.10.2.tgz#b41dbe9c928b03e8a83c9b0a973f60dc7c80c5be" - integrity sha512-qTnbvpada8qlEEyIPFwhTcF5Ns+k83fVlOSE8XvAtHkhcQ+okMnbvryVQBfP/ExRT1CRsQpYusdATR+FBJfrnQ== +vm2@3.11.3: + version "3.11.3" + resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.3.tgz#5323018a93ae2862c9d52b07b658ebb8d74903f0" + integrity sha512-DO1TTKuOc+veL11VNOvJwRab80mghFKE40Av3bl6pdXs11bdiDMuR73owy+dS2EsTZEvRUeBkkBuDVRjV/RgEw== dependencies: - acorn "^8.14.1" + acorn "^8.15.0" acorn-walk "^8.3.4" vsce@^1.79.5: From 60b69bfea2b616d4a7de4f86b241bf36b42cfaa3 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Fri, 22 May 2026 14:41:15 +0000 Subject: [PATCH 03/42] Bump DF_VERSION from 3.0.56 to 3.0.57 (#2173) --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 772b44cf9..40befac6a 100644 --- a/version.bzl +++ b/version.bzl @@ -1 +1 @@ -DF_VERSION = "3.0.56" +DF_VERSION = "3.0.57" From f1bfb2e51ef8d6012e5afa69a12a439dce893700 Mon Sep 17 00:00:00 2001 From: Edvin <56413069+SuchodolskiEdvin@users.noreply.github.com> Date: Mon, 25 May 2026 16:37:46 +0200 Subject: [PATCH 04/42] Support onSchemaChange for incremental tables (#2101) * Support onSchemaChange for incremental tables This change introduces support for the onSchemaChange option in incremental tables for the BigQuery adapter. It adds the incrementalSchemaChangeBody() strategy to handle schema changes. End-to-end tests have been created to verify the new functionality. * fix: Private functions moved after public functions * - Split onSchemaChange queries to sub-tasks - Split sql queries to sun-functions - Fixed function names * fix: Address feedback for PR #2101 and fix presubmit failures - Updated e2e tests - Added golden files - Consolidated merge and insert functions --- cli/api/BUILD | 6 +- cli/api/dbadapters/execution_sql.ts | 251 ++++++++++++++++-- cli/api/execution_sql_test.ts | 89 +++++++ cli/api/goldens/on_schema_change_extend.sql | 78 ++++++ cli/api/goldens/on_schema_change_fail.sql | 69 +++++ cli/api/goldens/on_schema_change_ignore.sql | 9 + .../goldens/on_schema_change_synchronize.sql | 98 +++++++ cli/index_run_e2e_test.ts | 163 ++++++++++++ 8 files changed, 742 insertions(+), 21 deletions(-) create mode 100644 cli/api/execution_sql_test.ts create mode 100644 cli/api/goldens/on_schema_change_extend.sql create mode 100644 cli/api/goldens/on_schema_change_fail.sql create mode 100644 cli/api/goldens/on_schema_change_ignore.sql create mode 100644 cli/api/goldens/on_schema_change_synchronize.sql diff --git a/cli/api/BUILD b/cli/api/BUILD index 02e7337fb..39bfea237 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -1,6 +1,6 @@ -load("//tools:ts_library.bzl", "ts_library") load("//testing:index.bzl", "ts_test_suite") load("//tools:node_modules.bzl", "node_modules") +load("//tools:ts_library.bzl", "ts_library") package(default_visibility = ["//visibility:public"]) @@ -62,6 +62,8 @@ ts_test_suite( "utils_test.ts", "commands/jit/rpc_test.ts", "dbadapters/bigquery_test.ts", + "execution_sql_test.ts", + "utils_test.ts", ], data = [ ":node_modules", @@ -69,7 +71,7 @@ ts_test_suite( "//test_credentials:bigquery.json", "@nodejs//:node", "@nodejs//:npm", - ], + ] + glob(["goldens/**"]), deps = [ ":api", "//core", diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index f226b9e91..33d1990e4 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -18,7 +18,8 @@ export class ExecutionSql { constructor( private readonly project: dataform.IProjectConfig, - private readonly dataformCoreVersion: string + private readonly dataformCoreVersion: string, + private readonly uniqueIdGenerator: () => string = () => Math.random().toString(36).substring(2) ) { this.CompilationSql = new CompilationSql(project, dataformCoreVersion); } @@ -121,6 +122,10 @@ from (${query}) as insertions`; return this.CompilationSql.resolveTarget(target); } + public getIncrementalQuery(table: dataform.ITable): string { + return this.where(table.incrementalQuery || table.query, table.where); + } + public publishTasks( table: dataform.ITable, runConfig: dataform.IRunConfig, @@ -141,23 +146,34 @@ from (${query}) as insertions`; if (!this.shouldWriteIncrementally(table, runConfig, tableMetadata)) { tasks.add(Task.statement(this.createOrReplace(table))); } else { - tasks.add( - Task.statement( - table.uniqueKey && table.uniqueKey.length > 0 - ? this.mergeInto( - table.target, - tableMetadata?.fields.map(f => f.name), - this.where(table.incrementalQuery || table.query, table.where), - table.uniqueKey, - table.bigquery && table.bigquery.updatePartitionFilter - ) - : this.insertInto( - table.target, - tableMetadata?.fields.map(f => f.name).map(column => `\`${column}\``), - this.where(table.incrementalQuery || table.query, table.where) - ) - ) - ); + const onSchemaChange = table.onSchemaChange ?? dataform.OnSchemaChange.IGNORE; + switch (onSchemaChange) { + case dataform.OnSchemaChange.FAIL: + case dataform.OnSchemaChange.EXTEND: + case dataform.OnSchemaChange.SYNCHRONIZE: + this.buildIncrementalSchemaChangeTasks(tasks, table); + // Fall through to run the static DML after the procedure alters the schema + case dataform.OnSchemaChange.IGNORE: + default: + tasks.add( + Task.statement( + table.uniqueKey && table.uniqueKey.length > 0 + ? this.mergeInto( + table.target, + tableMetadata?.fields.map(f => f.name), + this.getIncrementalQuery(table), + table.uniqueKey, + table.bigquery && table.bigquery.updatePartitionFilter + ) + : this.insertInto( + table.target, + tableMetadata?.fields.map(f => f.name).map(column => `\`${column}\``), + this.getIncrementalQuery(table) + ) + ) + ); + break; + } } } else { tasks.add(Task.statement(this.createOrReplace(table))); @@ -186,6 +202,203 @@ from (${query}) as insertions`; return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`; } + private buildIncrementalSchemaChangeTasks(tasks: Tasks, table: dataform.ITable) { + const uniqueId = this.uniqueIdGenerator(); + + const emptyTempTableTarget = { + ...table.target, + name: `${table.target.name}_df_temp_${uniqueId}_empty` + }; + + const procedureName = this.createProcedureName(table.target, uniqueId); + const procedureBody = this.incrementalSchemaChangeBody( + table, + this.resolveTarget(table.target), + emptyTempTableTarget + ); + + const createProcedureSql = `CREATE OR REPLACE PROCEDURE ${procedureName}() +OPTIONS(strict_mode=false) +BEGIN +${procedureBody} +END;`; + + const callProcedureSql = this.safeCallAndDropProcedure( + procedureName, + this.resolveTarget(emptyTempTableTarget) + ); + tasks.add(Task.statement(createProcedureSql)); + tasks.add(Task.statement(callProcedureSql)); + } + + private createProcedureName(target: dataform.ITarget, uniqueId: string): string { + return this.resolveTarget({ + ...target, + name: `df_osc_${uniqueId}` + }); + } + + private safeCallAndDropProcedure( + procedureName: string, + emptyTempTableName: string + ): string { + return ` +BEGIN + CALL ${procedureName}(); +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS ${emptyTempTableName}; + DROP PROCEDURE IF EXISTS ${procedureName}; + RAISE; +END; +DROP PROCEDURE IF EXISTS ${procedureName};`; + } + + private createEmptyTempTableSql(emptyTempTableName: string, query: string): string { + return ` +-- Create empty table to extract schema of new query. +CREATE OR REPLACE TABLE ${emptyTempTableName} AS ( + SELECT * FROM (${query}) AS insertions LIMIT 0 +);`; + } + + private compareSchemasSql( + target: dataform.ITarget, + emptyTempTableTarget: dataform.ITarget + ): string { + return ` +-- Compare schemas +DECLARE dataform_columns ARRAY; +DECLARE temp_table_columns ARRAY>; +DECLARE columns_added ARRAY>; +DECLARE columns_removed ARRAY; + +SET dataform_columns = ( + SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) + FROM \`${target.database}.${target.schema}.INFORMATION_SCHEMA.COLUMNS\` + WHERE table_name = '${target.name}' +); + +SET temp_table_columns = ( + SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) + FROM \`${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}.INFORMATION_SCHEMA.COLUMNS\` + WHERE table_name = '${emptyTempTableTarget.name}' +); + +SET columns_added = ( + SELECT IFNULL(ARRAY_AGG(column_info), []) + FROM UNNEST(temp_table_columns) AS column_info + WHERE column_info.column_name NOT IN UNNEST(dataform_columns) +); +SET columns_removed = ( + SELECT IFNULL(ARRAY_AGG(column_name), []) + FROM UNNEST(dataform_columns) AS column_name + WHERE column_name NOT IN (SELECT col.column_name FROM UNNEST(temp_table_columns) AS col) +);`; + } + + private applySchemaChangeStrategySql( + table: dataform.ITable, + qualifiedTargetTableName: string + ): string { + const onSchemaChange = table.onSchemaChange || dataform.OnSchemaChange.IGNORE; + let sql = ` +-- Apply schema change strategy (${dataform.OnSchemaChange[onSchemaChange]}).`; + + switch (onSchemaChange) { + case dataform.OnSchemaChange.FAIL: + sql += ` +IF ARRAY_LENGTH(columns_added) > 0 OR ARRAY_LENGTH(columns_removed) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Schema mismatch defined by on_schema_change = 'FAIL'. Added columns: %T, removed columns: %T", + columns_added, + columns_removed + ); +END IF; +`; + break; + case dataform.OnSchemaChange.EXTEND: + sql += ` +IF ARRAY_LENGTH(columns_removed) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Column removals are not allowed when on_schema_change = 'EXTEND'. Removed columns: %T", + columns_removed + ); +END IF; + +${this.alterTableAddColumnsSql(qualifiedTargetTableName)} +`; + break; + case dataform.OnSchemaChange.SYNCHRONIZE: + const uniqueKeys = table.uniqueKey || []; + sql += ` +DECLARE invalid_removed_columns ARRAY; +SET invalid_removed_columns = ( + SELECT IFNULL(ARRAY_AGG(col), []) FROM UNNEST(columns_removed) AS col WHERE col IN UNNEST(${JSON.stringify(uniqueKeys)}) +); + +IF ARRAY_LENGTH(invalid_removed_columns) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Cannot drop columns %T as they are part of the unique key for table ${qualifiedTargetTableName}", + invalid_removed_columns + ); +END IF; + +IF ARRAY_LENGTH(columns_removed) > 0 THEN + EXECUTE IMMEDIATE ( + "ALTER TABLE ${qualifiedTargetTableName} " || + ( + SELECT STRING_AGG(FORMAT("DROP COLUMN IF EXISTS %s", col), ", ") + FROM UNNEST(columns_removed) AS col + ) + ); +END IF; + +${this.alterTableAddColumnsSql(qualifiedTargetTableName)} +`; + break; + } + return sql; + } + + private alterTableAddColumnsSql(qualifiedTargetTableName: string): string { + return `IF ARRAY_LENGTH(columns_added) > 0 THEN + EXECUTE IMMEDIATE ( + "ALTER TABLE ${qualifiedTargetTableName} " || + ( + SELECT STRING_AGG(FORMAT("ADD COLUMN IF NOT EXISTS %s %s", column_info.column_name, column_info.data_type), ", ") + FROM UNNEST(columns_added) AS column_info + ) + ); +END IF;`; + } + + private cleanupSql(emptyTempTableName: string): string { + return ` +-- Cleanup temporary tables. +DROP TABLE IF EXISTS ${emptyTempTableName}; + `; + } + + private incrementalSchemaChangeBody( + table: dataform.ITable, + qualifiedTargetTableName: string, + emptyTempTableTarget: dataform.ITarget + ): string { + const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); + const query = this.getIncrementalQuery(table); + const statements: string[] = [ + this.createEmptyTempTableSql(emptyTempTableName, query), + this.compareSchemasSql( + table.target, + emptyTempTableTarget + ), + this.applySchemaChangeStrategySql(table, qualifiedTargetTableName), + this.cleanupSql(emptyTempTableName) + ]; + + return statements.join("\n\n"); + } + private createOrReplace(table: dataform.ITable) { const options = []; if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) { @@ -218,7 +431,7 @@ from (${query}) as insertions`; create or replace view ${this.resolveTarget(target)} as ${query}`; } - private mergeInto( + private mergeInto( target: dataform.ITarget, columns: string[], query: string, diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts new file mode 100644 index 000000000..4a8cf5568 --- /dev/null +++ b/cli/api/execution_sql_test.ts @@ -0,0 +1,89 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; + +import { ExecutionSql } from "df/cli/api/dbadapters/execution_sql"; +import { dataform } from "df/protos/ts"; +import { suite, test } from "df/testing"; + +suite("ExecutionSql with 'onSchemaChange'", () => { + const executionSql = new ExecutionSql( + { + defaultDatabase: "project-id", + defaultSchema: "dataset-id" + }, + "2.0.0", + () => "test_uuid" + ); + + const baseTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: { + database: "project-id", + schema: "dataset-id", + name: "incremental_on_schema_change" + }, + query: "select 1 as id, 'a' as field1", + incrementalQuery: "select 1 as id, 'a' as field1, 'new' as field2" + }; + + const tableMetadata: dataform.ITableMetadata = { + type: dataform.TableMetadata.Type.TABLE, + fields: [ + { + name: "id", + primitive: dataform.Field.Primitive.INTEGER + }, + { + name: "field1", + primitive: dataform.Field.Primitive.STRING + } + ] + }; + + test("generates procedure for FAIL strategy", () => { + const table = { + ...baseTable, + onSchemaChange: dataform.OnSchemaChange.FAIL + }; + const tasks = executionSql.publishTasks(table, { fullRefresh: false }, tableMetadata); + const procedureSql = tasks.build().map(t => t.statement).join("\n;\n"); + const expectedSql = fs.readFileSync("cli/api/goldens/on_schema_change_fail.sql", "utf8"); + expect(procedureSql).to.equal(expectedSql.trim()); + }); + + test("generates procedure for EXTEND strategy", () => { + const table = { + ...baseTable, + onSchemaChange: dataform.OnSchemaChange.EXTEND + }; + const tasks = executionSql.publishTasks(table, { fullRefresh: false }, tableMetadata); + const procedureSql = tasks.build().map(t => t.statement).join("\n;\n"); + const expectedSql = fs.readFileSync("cli/api/goldens/on_schema_change_extend.sql", "utf8"); + expect(procedureSql).to.equal(expectedSql.trim()); + }); + + test("generates procedure for SYNCHRONIZE strategy", () => { + const table = { + ...baseTable, + onSchemaChange: dataform.OnSchemaChange.SYNCHRONIZE, + uniqueKey: ["id"] + }; + const tasks = executionSql.publishTasks(table, { fullRefresh: false }, tableMetadata); + const procedureSql = tasks.build().map(t => t.statement).join("\n;\n"); + const expectedSql = fs.readFileSync("cli/api/goldens/on_schema_change_synchronize.sql", "utf8"); + expect(procedureSql).to.equal(expectedSql.trim()); + }); + + test("generates simple merge for IGNORE strategy", () => { + const table = { + ...baseTable, + onSchemaChange: dataform.OnSchemaChange.IGNORE, + uniqueKey: ["id"] + }; + const tasks = executionSql.publishTasks(table, { fullRefresh: false }, tableMetadata); + const procedureSql = tasks.build().map(t => t.statement).join("\n;\n"); + const expectedSql = fs.readFileSync("cli/api/goldens/on_schema_change_ignore.sql", "utf8"); + expect(procedureSql).to.equal(expectedSql.trim()); + }); +}); diff --git a/cli/api/goldens/on_schema_change_extend.sql b/cli/api/goldens/on_schema_change_extend.sql new file mode 100644 index 000000000..21f9558a8 --- /dev/null +++ b/cli/api/goldens/on_schema_change_extend.sql @@ -0,0 +1,78 @@ +CREATE OR REPLACE PROCEDURE `project-id.dataset-id.df_osc_test_uuid`() +OPTIONS(strict_mode=false) +BEGIN + +-- Create empty table to extract schema of new query. +CREATE OR REPLACE TABLE `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty` AS ( + SELECT * FROM (select 1 as id, 'a' as field1, 'new' as field2) AS insertions LIMIT 0 +); + + +-- Compare schemas +DECLARE dataform_columns ARRAY; +DECLARE temp_table_columns ARRAY>; +DECLARE columns_added ARRAY>; +DECLARE columns_removed ARRAY; + +SET dataform_columns = ( + SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change' +); + +SET temp_table_columns = ( + SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change_df_temp_test_uuid_empty' +); + +SET columns_added = ( + SELECT IFNULL(ARRAY_AGG(column_info), []) + FROM UNNEST(temp_table_columns) AS column_info + WHERE column_info.column_name NOT IN UNNEST(dataform_columns) +); +SET columns_removed = ( + SELECT IFNULL(ARRAY_AGG(column_name), []) + FROM UNNEST(dataform_columns) AS column_name + WHERE column_name NOT IN (SELECT col.column_name FROM UNNEST(temp_table_columns) AS col) +); + + +-- Apply schema change strategy (EXTEND). +IF ARRAY_LENGTH(columns_removed) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Column removals are not allowed when on_schema_change = 'EXTEND'. Removed columns: %T", + columns_removed + ); +END IF; + +IF ARRAY_LENGTH(columns_added) > 0 THEN + EXECUTE IMMEDIATE ( + "ALTER TABLE `project-id.dataset-id.incremental_on_schema_change` " || + ( + SELECT STRING_AGG(FORMAT("ADD COLUMN IF NOT EXISTS %s %s", column_info.column_name, column_info.data_type), ", ") + FROM UNNEST(columns_added) AS column_info + ) + ); +END IF; + + + +-- Cleanup temporary tables. +DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + +END +; +BEGIN + CALL `project-id.dataset-id.df_osc_test_uuid`(); +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; + RAISE; +END; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` +; +insert into `project-id.dataset-id.incremental_on_schema_change` +(`id`,`field1`) +select `id`,`field1` +from (select 1 as id, 'a' as field1, 'new' as field2) as insertions \ No newline at end of file diff --git a/cli/api/goldens/on_schema_change_fail.sql b/cli/api/goldens/on_schema_change_fail.sql new file mode 100644 index 000000000..ea8d90542 --- /dev/null +++ b/cli/api/goldens/on_schema_change_fail.sql @@ -0,0 +1,69 @@ +CREATE OR REPLACE PROCEDURE `project-id.dataset-id.df_osc_test_uuid`() +OPTIONS(strict_mode=false) +BEGIN + +-- Create empty table to extract schema of new query. +CREATE OR REPLACE TABLE `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty` AS ( + SELECT * FROM (select 1 as id, 'a' as field1, 'new' as field2) AS insertions LIMIT 0 +); + + +-- Compare schemas +DECLARE dataform_columns ARRAY; +DECLARE temp_table_columns ARRAY>; +DECLARE columns_added ARRAY>; +DECLARE columns_removed ARRAY; + +SET dataform_columns = ( + SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change' +); + +SET temp_table_columns = ( + SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change_df_temp_test_uuid_empty' +); + +SET columns_added = ( + SELECT IFNULL(ARRAY_AGG(column_info), []) + FROM UNNEST(temp_table_columns) AS column_info + WHERE column_info.column_name NOT IN UNNEST(dataform_columns) +); +SET columns_removed = ( + SELECT IFNULL(ARRAY_AGG(column_name), []) + FROM UNNEST(dataform_columns) AS column_name + WHERE column_name NOT IN (SELECT col.column_name FROM UNNEST(temp_table_columns) AS col) +); + + +-- Apply schema change strategy (FAIL). +IF ARRAY_LENGTH(columns_added) > 0 OR ARRAY_LENGTH(columns_removed) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Schema mismatch defined by on_schema_change = 'FAIL'. Added columns: %T, removed columns: %T", + columns_added, + columns_removed + ); +END IF; + + + +-- Cleanup temporary tables. +DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + +END +; +BEGIN + CALL `project-id.dataset-id.df_osc_test_uuid`(); +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; + RAISE; +END; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` +; +insert into `project-id.dataset-id.incremental_on_schema_change` +(`id`,`field1`) +select `id`,`field1` +from (select 1 as id, 'a' as field1, 'new' as field2) as insertions \ No newline at end of file diff --git a/cli/api/goldens/on_schema_change_ignore.sql b/cli/api/goldens/on_schema_change_ignore.sql new file mode 100644 index 000000000..0f9c016ba --- /dev/null +++ b/cli/api/goldens/on_schema_change_ignore.sql @@ -0,0 +1,9 @@ +merge `project-id.dataset-id.incremental_on_schema_change` T +using (select 1 as id, 'a' as field1, 'new' as field2 +) S +on T.id = S.id + +when matched then + update set `id` = S.id,`field1` = S.field1 +when not matched then + insert (`id`,`field1`) values (`id`,`field1`) \ No newline at end of file diff --git a/cli/api/goldens/on_schema_change_synchronize.sql b/cli/api/goldens/on_schema_change_synchronize.sql new file mode 100644 index 000000000..bcfc63eac --- /dev/null +++ b/cli/api/goldens/on_schema_change_synchronize.sql @@ -0,0 +1,98 @@ +CREATE OR REPLACE PROCEDURE `project-id.dataset-id.df_osc_test_uuid`() +OPTIONS(strict_mode=false) +BEGIN + +-- Create empty table to extract schema of new query. +CREATE OR REPLACE TABLE `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty` AS ( + SELECT * FROM (select 1 as id, 'a' as field1, 'new' as field2) AS insertions LIMIT 0 +); + + +-- Compare schemas +DECLARE dataform_columns ARRAY; +DECLARE temp_table_columns ARRAY>; +DECLARE columns_added ARRAY>; +DECLARE columns_removed ARRAY; + +SET dataform_columns = ( + SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change' +); + +SET temp_table_columns = ( + SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) + FROM `project-id.dataset-id.INFORMATION_SCHEMA.COLUMNS` + WHERE table_name = 'incremental_on_schema_change_df_temp_test_uuid_empty' +); + +SET columns_added = ( + SELECT IFNULL(ARRAY_AGG(column_info), []) + FROM UNNEST(temp_table_columns) AS column_info + WHERE column_info.column_name NOT IN UNNEST(dataform_columns) +); +SET columns_removed = ( + SELECT IFNULL(ARRAY_AGG(column_name), []) + FROM UNNEST(dataform_columns) AS column_name + WHERE column_name NOT IN (SELECT col.column_name FROM UNNEST(temp_table_columns) AS col) +); + + +-- Apply schema change strategy (SYNCHRONIZE). +DECLARE invalid_removed_columns ARRAY; +SET invalid_removed_columns = ( + SELECT IFNULL(ARRAY_AGG(col), []) FROM UNNEST(columns_removed) AS col WHERE col IN UNNEST(["id"]) +); + +IF ARRAY_LENGTH(invalid_removed_columns) > 0 THEN + RAISE USING MESSAGE = FORMAT( + "Cannot drop columns %T as they are part of the unique key for table `project-id.dataset-id.incremental_on_schema_change`", + invalid_removed_columns + ); +END IF; + +IF ARRAY_LENGTH(columns_removed) > 0 THEN + EXECUTE IMMEDIATE ( + "ALTER TABLE `project-id.dataset-id.incremental_on_schema_change` " || + ( + SELECT STRING_AGG(FORMAT("DROP COLUMN IF EXISTS %s", col), ", ") + FROM UNNEST(columns_removed) AS col + ) + ); +END IF; + +IF ARRAY_LENGTH(columns_added) > 0 THEN + EXECUTE IMMEDIATE ( + "ALTER TABLE `project-id.dataset-id.incremental_on_schema_change` " || + ( + SELECT STRING_AGG(FORMAT("ADD COLUMN IF NOT EXISTS %s %s", column_info.column_name, column_info.data_type), ", ") + FROM UNNEST(columns_added) AS column_info + ) + ); +END IF; + + + +-- Cleanup temporary tables. +DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + +END +; +BEGIN + CALL `project-id.dataset-id.df_osc_test_uuid`(); +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; + DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; + RAISE; +END; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` +; +merge `project-id.dataset-id.incremental_on_schema_change` T +using (select 1 as id, 'a' as field1, 'new' as field2 +) S +on T.id = S.id + +when matched then + update set `id` = S.id,`field1` = S.field1 +when not matched then + insert (`id`,`field1`) values (`id`,`field1`) \ No newline at end of file diff --git a/cli/index_run_e2e_test.ts b/cli/index_run_e2e_test.ts index 6accdfcb9..da19b4af8 100644 --- a/cli/index_run_e2e_test.ts +++ b/cli/index_run_e2e_test.ts @@ -646,4 +646,167 @@ select 2 ] }]); }); + + suite("onSchemaChange", ({ beforeEach }) => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const uniqueDataset = `dataform_e2e_osc_${Math.random().toString(36).substring(7)}`; + + beforeEach("setup test project", async () => { + const npmCacheDir = tmpDirFixture.createNewTmpDir(); + const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); + const packageJsonPath = path.join(projectDir, "package.json"); + + await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + ); + + const workflowSettings = dataform.WorkflowSettings.create( + loadYaml(fs.readFileSync(workflowSettingsPath, "utf8")) + ); + delete workflowSettings.dataformCoreVersion; + workflowSettings.defaultDataset = uniqueDataset; + fs.writeFileSync(workflowSettingsPath, dumpYaml(workflowSettings)); + + fs.writeFileSync( + packageJsonPath, + `{ + "dependencies":{ + "@dataform/core": "${version}" + } +}` + ); + await getProcessResult( + execFile(npmPath, [ + "install", + "--prefix", + projectDir, + "--cache", + npmCacheDir, + corePackageTarPath + ]) + ); + + fs.ensureFileSync(path.join(projectDir, "definitions", "setup_table.sqlx")); + fs.writeFileSync( + path.join(projectDir, "definitions", "setup_table.sqlx"), + ` +config { + type: "operations" +} +CREATE OR REPLACE TABLE \`\${dataform.projectConfig.defaultDatabase}.\${dataform.projectConfig.defaultSchema}.example_incremental\` AS SELECT 1 AS id, 'old' AS field1 +` + ); + + fs.ensureFileSync(path.join(projectDir, "definitions", "example_incremental.sqlx")); + fs.writeFileSync( + path.join(projectDir, "definitions", "example_incremental.sqlx"), + ` +config { + type: "incremental", + onSchemaChange: "EXTEND" +} +SELECT 1 as id, 'new' as field1, 'new2' as field2 +` + ); + + fs.ensureFileSync(path.join(projectDir, "definitions", "teardown_schema.sqlx")); + fs.writeFileSync( + path.join(projectDir, "definitions", "teardown_schema.sqlx"), + ` +config { + type: "operations" +} +DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.projectConfig.defaultSchema}\` CASCADE +` + ); + }); + + test("generates dynamic SQL for EXTEND when table exists in BigQuery", async () => { + try { + // Run setup operation to create the table in BigQuery. + // Dataform will automatically create the uniqueDataset schema. + await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "run", + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--actions=setup_table" + ]) + ); + + // Run the incremental table in dry-run mode. + // Dataform will detect the table exists and generate the dynamic procedural SQL. + const runResult = await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "run", + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--dry-run", + "--json", + "--actions=example_incremental" + ]) + ); + + expect(runResult.exitCode).equals(0); + const executionGraph = JSON.parse(runResult.stdout); + const statement = executionGraph.actions[0].tasks[0].statement; + + const expectedRunResult = { + projectConfig: { + warehouse: "bigquery", + defaultSchema: uniqueDataset, + assertionSchema: "dataform_assertions", + defaultDatabase: DEFAULT_DATABASE, + defaultLocation: DEFAULT_LOCATION + }, + runConfig: { + actions: ["example_incremental"], + fullRefresh: false + }, + actions: [ + { + fileName: "definitions/example_incremental.sqlx", + hermeticity: "NON_HERMETIC", + tableType: "incremental", + target: { + database: DEFAULT_DATABASE, + name: "example_incremental", + schema: uniqueDataset + }, + tasks: [ + { + statement, + type: "statement" + } + ], + type: "table" + } + ], + warehouseState: executionGraph.warehouseState + }; + + expect(executionGraph).deep.equals(expectedRunResult); + expect(statement).to.include("CREATE OR REPLACE PROCEDURE"); + expect(statement).to.include("Column removals are not allowed when on_schema_change = 'EXTEND'."); + expect(statement).to.include("ALTER TABLE"); + expect(statement).to.include("ADD COLUMN IF NOT EXISTS"); + } finally { + // Teardown the schema completely, regardless of test success or failure. + await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "run", + projectDir, + "--credentials", + CREDENTIALS_PATH, + "--actions=teardown_schema" + ]) + ); + } + }); + }); }); From c8d5d6a34d672617cc28e18dd3b0598400477826 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 26 May 2026 07:04:03 +0000 Subject: [PATCH 05/42] feat: implement .jitCode() support for Assertion actions (#2170) - Add `jit_code` field to the `Assertion` message in `protos/core.proto` - Fix missing protobuf imports and configuration in `protos/BUILD` - Introduce `JitAssertionResult` type alias and `jitCode` method to `Assertion` builder (`core/actions/assertion.ts`) - Add strictly typed `jitContextable` parameter to `Session.sqlxAction` in `core/session.ts` - Update assertion `compile()` to handle `jitCode` execution bypassing standard string evaluation - Parse and apply `jitContextable` safely for assertions in `core/session.ts` - Add unit test coverage in `core/main_test.ts` to verify compilation output --- core/actions/assertion.ts | 37 ++++++++++++++++++++++-- core/main_test.ts | 59 +++++++++++++++++++++++++++++++++++++++ protos/core.proto | 2 ++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts index e638e8b37..34cfa48b8 100644 --- a/core/actions/assertion.ts +++ b/core/actions/assertion.ts @@ -1,6 +1,6 @@ import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos"; import { ActionBuilder } from "df/core/actions"; -import { IActionContext, Resolvable } from "df/core/contextables"; +import { IActionContext, JitContextable, Resolvable } from "df/core/contextables"; import * as Path from "df/core/path"; import { Session } from "df/core/session"; import { @@ -32,6 +32,9 @@ interface ILegacyAssertionConfig extends dataform.ActionConfig.AssertionConfig { /** @hidden */ export type AContextable = T | ((ctx: AssertionContext) => T); +/** JiT compilation stage result for assertions. */ +export type JitAssertionResult = string; + /** * An assertion is a data quality test query that finds rows that violate one or more conditions * specified in the query. If the query returns any rows, the assertion fails. @@ -90,6 +93,9 @@ export class Assertion extends ActionBuilder { /** @hidden We delay contextification until the final compile step, so hold these here for now. */ private contextableQuery: AContextable; + /** @hidden */ + private contextableJitCode: JitContextable | undefined; + /** @hidden */ constructor(session?: Session, unverifiedConfig?: any, configPath?: string) { super(session); @@ -166,6 +172,15 @@ export class Assertion extends ActionBuilder { return this; } + public jitCode(jitCode: JitContextable) { + if (!this.proto.actionDescriptor) { + this.proto.actionDescriptor = {}; + } + this.proto.actionDescriptor.compilationMode = dataform.ActionCompilationMode.ACTION_COMPILATION_MODE_JIT; + this.contextableJitCode = jitCode; + return this; + } + /** * @deprecated Deprecated in favor of * [AssertionConfig.dependencies](configs#dataform-ActionConfig-AssertionConfig). @@ -293,8 +308,24 @@ export class Assertion extends ActionBuilder { public compile() { const context = new AssertionContext(this); - this.proto.query = context.apply(this.contextableQuery); - validateQueryString(this.session, this.proto.query, this.proto.fileName); + if (this.contextableJitCode && this.contextableQuery) { + this.session.compileError( + new Error("Assertion may set either .jitCode() or .query(), but not both."), + this.proto.fileName, + this.proto.target + ); + return this.proto; + } + + if (this.contextableJitCode) { + if (!this.proto.actionDescriptor) { + this.proto.actionDescriptor = {}; + } + this.proto.jitCode = this.contextableJitCode.toString(); + } else { + this.proto.query = context.apply(this.contextableQuery); + validateQueryString(this.session, this.proto.query, this.proto.fileName); + } return verifyObjectMatchesProto( dataform.Assertion, diff --git a/core/main_test.ts b/core/main_test.ts index 1b53e0e91..c23f62f2e 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -1537,6 +1537,65 @@ assert("name", { ]); }); + test("jitCode correctly populates the jitCode field", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/assert.js"), + ` +assert("name").jitCode(ctx => "jit");` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([ + { + actionDescriptor: { + compilationMode: "ACTION_COMPILATION_MODE_JIT" + }, + canonicalTarget: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + }, + fileName: "definitions/assert.js", + jitCode: 'ctx => "jit"', + target: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + } + } + ]); + }); + + test("fails when both jitCode and query are set on an assertion", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/assert.js"), + ` +assert("name").query("SELECT 1").jitCode(ctx => "jit");` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + "Assertion may set either .jitCode() or .query(), but not both." + ]); + }); + test("assert API returns disabled assertions when disableAssertions is true", () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( diff --git a/protos/core.proto b/protos/core.proto index 663332019..91657f3cd 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -260,6 +260,8 @@ message Assertion { // Only present for auto assertions. Target parent_action = 15; + string jit_code = 16; + // Generated. string file_name = 7; From bbbc19266eb5fefd634944bc35f076dfebffb160 Mon Sep 17 00:00:00 2001 From: Edvin <56413069+SuchodolskiEdvin@users.noreply.github.com> Date: Tue, 26 May 2026 14:30:03 +0200 Subject: [PATCH 06/42] fix: Resolve presubmit failure introduced by PR #2101 - extend onSchemaChange e2e test timeout (#2175) --- cli/index_run_e2e_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/index_run_e2e_test.ts b/cli/index_run_e2e_test.ts index da19b4af8..8a839f693 100644 --- a/cli/index_run_e2e_test.ts +++ b/cli/index_run_e2e_test.ts @@ -721,7 +721,7 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p ); }); - test("generates dynamic SQL for EXTEND when table exists in BigQuery", async () => { + test("generates dynamic SQL for EXTEND when table exists in BigQuery", { timeout: 120000 }, async () => { try { // Run setup operation to create the table in BigQuery. // Dataform will automatically create the uniqueDataset schema. From 62b18ba1e20c477b6d40e8d6a5a76dc0a6fb3060 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 26 May 2026 13:33:34 +0000 Subject: [PATCH 07/42] Bump DF_VERSION from 3.0.57 to 3.0.58 (#2176) --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 40befac6a..ecee4ae76 100644 --- a/version.bzl +++ b/version.bzl @@ -1 +1 @@ -DF_VERSION = "3.0.57" +DF_VERSION = "3.0.58" From 8d43204dfee4beed105f60424e56ea1f4345722d Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 26 May 2026 14:19:00 +0000 Subject: [PATCH 08/42] Guard workflow_settings.yaml and dataform.json require with existence check (#2178) --- cli/vm/compile.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 7faefb467..39e2e7d7d 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -77,10 +77,21 @@ export function compile(compileConfig: dataform.ICompileConfig) { throw new Error("@dataform/core ^3.0.0 required."); } + const hasWorkflowSettingsYaml = fs.existsSync( + path.join(compileConfig.projectDir, "workflow_settings.yaml") + ); + const hasDataformJson = fs.existsSync( + path.join(compileConfig.projectDir, "dataform.json") + ); + return userCodeVm.run( ` - global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed:", e); } })(); - global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); + ${hasWorkflowSettingsYaml + ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' + : ''} + ${hasDataformJson + ? 'global.dataformJson = require("./dataform.json");' + : ''} return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}") `, vmIndexFileName From 32269401f86e6d553edae12a75969e2437318ea0 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Thu, 28 May 2026 09:33:50 +0000 Subject: [PATCH 09/42] Refactor CompileChildProcess to use BaseWorker (#2179) --- cli/api/commands/base_worker.ts | 10 +++-- cli/api/commands/compile.ts | 80 ++++++++------------------------- cli/vm/compile.ts | 9 +++- tests/api/projects.spec.ts | 7 ++- 4 files changed, 39 insertions(+), 67 deletions(-) diff --git a/cli/api/commands/base_worker.ts b/cli/api/commands/base_worker.ts index a188ddbc9..f1c420722 100644 --- a/cli/api/commands/base_worker.ts +++ b/cli/api/commands/base_worker.ts @@ -15,6 +15,7 @@ export abstract class BaseWorker { return new Promise((resolve, reject) => { let completed = false; + let booted = false; const terminate = (fn: () => void) => { if (completed) { @@ -22,7 +23,7 @@ export abstract class BaseWorker { } completed = true; clearTimeout(timeout); - child.kill(); + child.kill("SIGKILL"); fn(); }; @@ -34,7 +35,10 @@ export abstract class BaseWorker { child.on("message", (message: any) => { if (message.type === "worker_booted") { - onBoot(child); + if (!booted) { + booted = true; + onBoot(child); + } return; } onMessage(message, child, (res) => terminate(() => resolve(res)), (err) => terminate(() => reject(err))); @@ -57,7 +61,7 @@ export abstract class BaseWorker { } private resolveScript() { - const pathsToTry = [this.loaderPath, "./worker_bundle.js"]; + const pathsToTry = ["./worker_bundle.js", this.loaderPath]; for (const p of pathsToTry) { try { return require.resolve(p); diff --git a/cli/api/commands/compile.ts b/cli/api/commands/compile.ts index ac863a9a3..e4649be4a 100644 --- a/cli/api/commands/compile.ts +++ b/cli/api/commands/compile.ts @@ -1,11 +1,13 @@ -import { ChildProcess, exec, fork } from "child_process"; +import { exec } from "child_process"; import * as fs from "fs-extra"; import * as path from "path"; import * as tmp from "tmp"; import { promisify } from "util"; +import { BaseWorker } from "df/cli/api/commands/base_worker"; import { MISSING_CORE_VERSION_ERROR } from "df/cli/api/commands/install"; import { readConfigFromWorkflowSettings } from "df/cli/api/utils"; +import { DEFAULT_COMPILATION_TIMEOUT_MILLIS } from "df/cli/api/utils/constants"; import { coerceAsError } from "df/common/errors/errors"; import { decode64 } from "df/common/protos"; import { dataform } from "df/protos/ts"; @@ -77,7 +79,7 @@ export async function compile( const { stdout, stderr } = await promisify(exec)(npmCommand, { cwd: temporaryProjectPath }); - + if (compileConfig.verbose) { print(`NPM HTTP Logs:\n${stderr}\n`); print(`NPM install completed in ${performance.now() - npmStartTime}ms\n`); @@ -86,7 +88,7 @@ export async function compile( compileConfig.projectDir = temporaryProjectPath; } - const result = await CompileChildProcess.forkProcess().compile(compileConfig); + const result = await new CompileChildProcess().compile(compileConfig); const decodedResult = decode64(dataform.CoreExecutionResponse, result); compiledGraph = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph); @@ -98,68 +100,24 @@ export async function compile( return compiledGraph; } -export class CompileChildProcess { - public static forkProcess() { - // Runs the worker_bundle script we generate for the package (see packages/@dataform/cli/BUILD) - // if it exists, otherwise run the bazel compile loader target. - const findForkScript = () => { - try { - const workerBundlePath = require.resolve("./worker_bundle.js"); - return workerBundlePath; - } catch (e) { - return require.resolve("../../vm/compile_loader"); - } - }; - const forkScript = findForkScript(); - return new CompileChildProcess( - fork(require.resolve(forkScript), [], { stdio: [0, 1, 2, "ipc", "pipe"] }) - ); - } - private readonly childProcess: ChildProcess; - - constructor(childProcess: ChildProcess) { - this.childProcess = childProcess; +export class CompileChildProcess extends BaseWorker { + constructor() { + super(path.resolve(__dirname, "../../vm/compile_loader")); } public async compile(compileConfig: dataform.ICompileConfig) { - const compileInChildProcess = new Promise(async (resolve, reject) => { - this.childProcess.on("error", (e: Error) => reject(coerceAsError(e))); - - this.childProcess.on("message", (messageOrError: string | Error) => { - if (typeof messageOrError === "string") { - resolve(messageOrError); - return; - } - reject(coerceAsError(messageOrError)); - }); - - this.childProcess.on("close", exitCode => { - if (exitCode !== 0) { - reject(new Error(`Compilation child process exited with exit code ${exitCode}.`)); + const timeoutValue = compileConfig.timeoutMillis || DEFAULT_COMPILATION_TIMEOUT_MILLIS; + + return await this.runWorker( + timeoutValue, + child => child.send(compileConfig), + (message, child, resolve, reject) => { + if (typeof message === "string") { + resolve(message); + } else { + reject(coerceAsError(message)); } - }); - - // Trigger the child process to start compiling. - this.childProcess.send(compileConfig); - }); - let timer; - const timeout = new Promise( - (resolve, reject) => - (timer = setTimeout( - () => reject(new CompilationTimeoutError("Compilation timed out")), - compileConfig.timeoutMillis || 5000 - )) - ); - try { - await Promise.race([timeout, compileInChildProcess]); - return await compileInChildProcess; - } finally { - if (!this.childProcess.killed) { - this.childProcess.kill("SIGKILL"); } - if (timer) { - clearTimeout(timer); - } - } + ); } } diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 39e2e7d7d..d4001222d 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -99,7 +99,11 @@ export function compile(compileConfig: dataform.ICompileConfig) { } export function listenForCompileRequest() { - process.on("message", (compileConfig: dataform.ICompileConfig) => { + process.on("message", (compileConfig: dataform.ICompileConfig & { type?: string }) => { + // JiT messages are handled by handleJitRequest in worker.ts; skip them here. + if ((compileConfig as { type?: string })?.type === "jit_compile") { + return; + } try { const compiledResult = compile(compileConfig); process.send(compiledResult); @@ -114,6 +118,9 @@ export function listenForCompileRequest() { } if (require.main === module) { + if (process.send) { + process.send({ type: "worker_booted" }); + } listenForCompileRequest(); } diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts index 82c62f802..f70bc85ab 100644 --- a/tests/api/projects.spec.ts +++ b/tests/api/projects.spec.ts @@ -694,10 +694,13 @@ suite("examples", () => { test("times out after timeout period during compilation", async () => { try { - await compile({ projectDir: "tests/api/projects/never_finishes_compiling" }); + await compile({ + projectDir: "tests/api/projects/never_finishes_compiling", + timeoutMillis: 1000 + }); fail("Compilation timeout Error expected."); } catch (e) { - expect(e.message).to.equal("Compilation timed out"); + expect(e.message).to.equal("Worker timed out after 1 seconds"); } }); From b50f212421678ab2848d7988208c3a79c4a6376f Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Thu, 28 May 2026 10:50:37 +0000 Subject: [PATCH 10/42] Rename worker timeout error to "Compilation timed out" (#2186) --- cli/api/commands/base_worker.ts | 2 +- cli/api/commands/compile.ts | 4 ++-- tests/api/projects.spec.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/api/commands/base_worker.ts b/cli/api/commands/base_worker.ts index f1c420722..e2be64b38 100644 --- a/cli/api/commands/base_worker.ts +++ b/cli/api/commands/base_worker.ts @@ -29,7 +29,7 @@ export abstract class BaseWorker { const timeout = setTimeout(() => { terminate(() => - reject(new Error(`Worker timed out after ${timeoutMillis / 1000} seconds`)) + reject(new Error(`Compilation timed out after ${timeoutMillis / 1000} seconds`)) ); }, timeoutMillis); diff --git a/cli/api/commands/compile.ts b/cli/api/commands/compile.ts index e4649be4a..38d32592a 100644 --- a/cli/api/commands/compile.ts +++ b/cli/api/commands/compile.ts @@ -114,9 +114,9 @@ export class CompileChildProcess extends BaseWorker { (message, child, resolve, reject) => { if (typeof message === "string") { resolve(message); - } else { - reject(coerceAsError(message)); + return; } + reject(coerceAsError(message)); } ); } diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts index f70bc85ab..7bff903d7 100644 --- a/tests/api/projects.spec.ts +++ b/tests/api/projects.spec.ts @@ -700,7 +700,7 @@ suite("examples", () => { }); fail("Compilation timeout Error expected."); } catch (e) { - expect(e.message).to.equal("Worker timed out after 1 seconds"); + expect(e.message).to.equal("Compilation timed out after 1 seconds"); } }); From f2c84314370ec8e24eb25d58794655440f18a3b4 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Thu, 28 May 2026 13:12:43 +0000 Subject: [PATCH 11/42] Extract task creation into executionSql helpers (#2180) --- cli/api/commands/build.ts | 12 +++--------- cli/api/dbadapters/execution_sql.ts | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/cli/api/commands/build.ts b/cli/api/commands/build.ts index 99538381a..cfcf07d02 100644 --- a/cli/api/commands/build.ts +++ b/cli/api/commands/build.ts @@ -82,9 +82,7 @@ export class Builder { ...this.toPartialExecutionAction(table), type: "table", tableType: utils.tableTypeEnumToString(table.enumType), - tasks: table.disabled - ? [] - : this.executionSql.publishTasks(table, runConfig, tableMetadata).build(), + tasks: this.executionSql.createTableTasks(table, runConfig, tableMetadata), hermeticity: table.hermeticity || dataform.ActionHermeticity.HERMETIC }; } @@ -93,9 +91,7 @@ export class Builder { return { ...this.toPartialExecutionAction(operation), type: "operation", - tasks: operation.disabled - ? [] - : operation.queries.map(statement => ({ type: "statement", statement })), + tasks: this.executionSql.createOperationTasks(operation), hermeticity: operation.hermeticity || dataform.ActionHermeticity.NON_HERMETIC }; } @@ -104,9 +100,7 @@ export class Builder { return { ...this.toPartialExecutionAction(assertion), type: "assertion", - tasks: assertion.disabled - ? [] - : this.executionSql.assertTasks(assertion, this.prunedGraph.projectConfig).build(), + tasks: this.executionSql.createAssertionTasks(assertion), hermeticity: assertion.hermeticity || dataform.ActionHermeticity.HERMETIC }; } diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 33d1990e4..3745e7a2f 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -184,6 +184,26 @@ from (${query}) as insertions`; return tasks.concatenate(); } + public createTableTasks( + table: dataform.ITable, + runConfig: dataform.IRunConfig, + tableMetadata?: dataform.ITableMetadata + ): dataform.IExecutionTask[] { + return table.disabled ? [] : this.publishTasks(table, runConfig, tableMetadata).build(); + } + + public createOperationTasks(operation: dataform.IOperation): dataform.IExecutionTask[] { + return operation.disabled + ? [] + : operation.queries.map(statement => + dataform.ExecutionTask.create({ type: "statement", statement }) + ); + } + + public createAssertionTasks(assertion: dataform.IAssertion): dataform.IExecutionTask[] { + return assertion.disabled ? [] : this.assertTasks(assertion, this.project).build(); + } + public assertTasks( assertion: dataform.IAssertion, projectConfig: dataform.IProjectConfig, From a2f8d700a516219e871670d3680f23af897fbf97 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 2 Jun 2026 10:20:16 +0000 Subject: [PATCH 12/42] Add JiT compilation support for assertion actions (#2193) Adds JIT_COMPILATION_TARGET_TYPE_ASSERTION alongside the existing TABLE / OPERATION / INCREMENTAL_TABLE target types, plus a matching JitAssertionResult { string query } proto and a jitCompileAssertion() dispatcher in core/jit_compiler.ts. Assertions reuse SqlActionJitContext since AssertionContext implements IActionContext, and the result is a single SELECT query string (matching the existing JitAssertionResult = string alias in core/actions/assertion.ts). --- core/jit_compiler.ts | 16 ++++++++++++++++ core/jit_compiler_test.ts | 29 +++++++++++++++++++++++++++++ protos/jit.proto | 9 +++++++++ 3 files changed, 54 insertions(+) diff --git a/core/jit_compiler.ts b/core/jit_compiler.ts index c79935283..780b076e3 100644 --- a/core/jit_compiler.ts +++ b/core/jit_compiler.ts @@ -1,5 +1,6 @@ import * as $protobuf from "protobufjs"; +import { JitAssertionResult } from "df/core/actions/assertion"; import { JitOperationResult } from "df/core/actions/operation"; import { JitTableResult } from "df/core/actions/table"; import { IActionContext, ITableContext, JitContext } from "df/core/contextables"; @@ -63,6 +64,18 @@ function jitCompileTable( return mainBody(jctx).then(makeJitTableResult); } +function jitCompileAssertion( + request: dataform.IJitCompilationRequest, + adapter: dataform.DbAdapter, +): Promise { + const mainBody = makeMainBody(request.jitCode); + + const jctx: JitContext = new SqlActionJitContext( + adapter, request, + ); + return mainBody(jctx).then(query => dataform.JitAssertionResult.create({ query })); +} + function jitCompileIncrementalTable( request: dataform.IJitCompilationRequest, adapter: dataform.DbAdapter, @@ -110,6 +123,9 @@ export function jitCompile(request: dataform.IJitCompilationRequest, rpcCallback case dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE: return jitCompileIncrementalTable(request, dbAdapter).then( incrementalTable => dataform.JitCompilationResponse.create({ incrementalTable })); + case dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_ASSERTION: + return jitCompileAssertion(request, dbAdapter).then( + assertion => dataform.JitCompilationResponse.create({ assertion })); default: throw new Error(`Unrecognized compilation target type: ${request.compilationTargetType}`); } diff --git a/core/jit_compiler_test.ts b/core/jit_compiler_test.ts index 9d252e051..a45958f53 100644 --- a/core/jit_compiler_test.ts +++ b/core/jit_compiler_test.ts @@ -120,6 +120,35 @@ suite("jit_compiler", () => { }); }); + suite("jitCompileAssertion", () => { + test("compiles assertion returning string", async () => { + const request = dataform.JitCompilationRequest.create({ + jitCode: `async (ctx) => "SELECT * FROM t WHERE invalid"`, + target, + jitData: {}, + compilationTargetType: dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_ASSERTION, + }); + const result = await jitCompile(request, rpcCallback); + expect(result.assertion.query).to.equal("SELECT * FROM t WHERE invalid"); + }); + + test("compiles assertion using context", async () => { + const request = dataform.JitCompilationRequest.create({ + jitCode: `async (ctx) => \`SELECT * FROM \${ctx.ref('other')} WHERE invalid\``, + target, + jitData: {}, + dependencies: [dataform.Target.create({ + database: "db", + schema: "schema", + name: "other", + })], + compilationTargetType: dataform.JitCompilationTargetType.JIT_COMPILATION_TARGET_TYPE_ASSERTION, + }); + const result = await jitCompile(request, rpcCallback); + expect(result.assertion.query).to.equal("SELECT * FROM `db.schema.other` WHERE invalid"); + }); + }); + suite("jitCompileIncrementalTable", () => { test("compiles incremental table", async () => { const request = dataform.JitCompilationRequest.create({ diff --git a/protos/jit.proto b/protos/jit.proto index e86e6ba2d..ca0d0707e 100644 --- a/protos/jit.proto +++ b/protos/jit.proto @@ -108,6 +108,8 @@ enum JitCompilationTargetType { JIT_COMPILATION_TARGET_TYPE_OPERATION = 2; // Incremental table target. JIT_COMPILATION_TARGET_TYPE_INCREMENTAL_TABLE = 3; + // Assertion target. + JIT_COMPILATION_TARGET_TYPE_ASSERTION = 4; } // JiT compilation request. @@ -134,6 +136,7 @@ message JitCompilationResponse { JitTableResult table = 1; JitOperationResult operation = 2; JitIncrementalTableResult incremental_table = 3; + JitAssertionResult assertion = 4; } } @@ -161,3 +164,9 @@ message JitOperationResult { // Sequence of SQL operations. repeated string queries = 1; } + +// JiT compilation result for assertion actions. +message JitAssertionResult { + // SQL Select query that returns rows iff the assertion fails. + string query = 1; +} From 4ad875e275caf419b5af8feb367605648626d132 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 2 Jun 2026 10:20:59 +0000 Subject: [PATCH 13/42] Fix caller-file error (#2177) and enforce Core/CLI version match (#2191) vm2 3.11.3 strips file paths from V8 CallSite objects inside the sandbox, so `getCallerFile()` in @dataform/core can no longer resolve the current file from the stack. Track it via a host-side stack exposed through `__df_enter`/`__df_exit`/`__df_current` sandbox helpers and read it through a getter on `global.__dataform_current_file`. For @dataform/core <= 3.0.56 (no global fallback in getCallerFile), patch the bundle text at load time to consult the global. Gated on the installed Core version so newer bundles are never touched. Add a CLI/Core compatibility check: reject installed Core whose major differs from the CLI's, or whose minor is below the CLI's minor. Resolve the installed Core's version by running `require("@dataform/core").version` inside the index-generator vm so any install layout (package.json, workflow_settings.yaml stateless install, JiT) reports the version actually loaded. The error tells the user the exact `dataformCoreVersion:` line to set in `workflow_settings.yaml`. Skipped when the CLI version can't be determined (unbundled local dev). Mirror the same wrapper pattern in `testing/run_core.ts`. Add `compile rejects @dataform/core with incompatible version` test in `cli/index_compile_test.ts` that drives the workflow_settings.yaml stateless install path with dataformCoreVersion: "2.9.0" (latest 2.x on the registry) to exercise the real `npm i` install flow. Add `compile succeeds with @dataform/core <= 3.0.56 via caller-file shim` test that installs @dataform/core@3.0.50 and asserts the compiled action's `fileName` matches the source path, end-to-end proving the bundle-text patch and host-side file stack restore getCallerFile under vm2 3.11.3. --- cli/index_compile_test.ts | 65 +++++++++++++++++++++++ cli/vm/compile.ts | 108 ++++++++++++++++++++++++++++++-------- testing/run_core.ts | 18 +++++-- 3 files changed, 165 insertions(+), 26 deletions(-) diff --git a/cli/index_compile_test.ts b/cli/index_compile_test.ts index c4966c5a7..4f82799b7 100644 --- a/cli/index_compile_test.ts +++ b/cli/index_compile_test.ts @@ -65,6 +65,71 @@ suite("compile command", ({ afterEach }) => { ); }); + test("compile rejects @dataform/core with incompatible version", async () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // dataformCoreVersion in workflow_settings.yaml triggers the stateless + // install path (compile.ts copies to a tmp dir and runs `npm i`), so the + // test exercises the same flow real users hit. 2.9.0 is the latest 2.x on + // the registry; its major (2) is incompatible with the current CLI (3.x). + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + dumpYaml( + dataform.WorkflowSettings.create({ + defaultProject: "dataform", + dataformCoreVersion: "2.9.0" + }) + ) + ); + + // npm needs a writable cache; ~/.npm is read-only in the bazel sandbox. + const npmCacheDir = tmpDirFixture.createNewTmpDir(); + const stderr = ( + await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir], { + env: { ...process.env, NPM_CONFIG_CACHE: npmCacheDir } + }) + ) + ).stderr; + expect(stderr).contains("@dataform/core 2.9.0 is not compatible with @dataform/cli"); + expect(stderr).contains("matching major.minor"); + expect(stderr).contains("Set `dataformCoreVersion:"); + }); + + test("compile succeeds with @dataform/core <= 3.0.56 via caller-file shim", async () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // 3.0.50 predates 3.0.57, which is when @dataform/core started reading + // global.__dataform_current_file as a fallback in getCallerFile(). The + // compile path text-patches the bundle to add that fallback; this test + // proves the patch + host-side file stack drive a real action's + // fileName from inside vm2 3.11.3's path-stripped sandbox. + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + dumpYaml({ + defaultProject: DEFAULT_DATABASE, + defaultLocation: DEFAULT_LOCATION, + defaultDataset: "dataform", + dataformCoreVersion: "3.0.50" + }) + ); + fs.ensureFileSync(path.join(projectDir, "definitions", "example.sqlx")); + fs.writeFileSync( + path.join(projectDir, "definitions", "example.sqlx"), + `config { type: "table" }\nSELECT 1 AS id` + ); + + const npmCacheDir = tmpDirFixture.createNewTmpDir(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--json"], { + env: { ...process.env, NPM_CONFIG_CACHE: npmCacheDir } + }) + ); + + expect(result.exitCode, `compile failed: ${result.stderr}`).equals(0); + const compiled = JSON.parse(result.stdout); + expect(compiled.tables).to.have.lengthOf(1); + expect(compiled.tables[0].fileName).equals("definitions/example.sqlx"); + }); + ["package.json", "package-lock.json", "node_modules"].forEach(npmFile => { test(`compile throws an error when dataformCoreVersion in workflow_settings.yaml and ${npmFile} is present`, async () => { const projectDir = tmpDirFixture.createNewTmpDir(); diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index d4001222d..5ae3fe93d 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -9,11 +9,10 @@ import { dataform } from "df/protos/ts"; export function compile(compileConfig: dataform.ICompileConfig) { compileConfig.projectDir = fs.realpathSync(path.resolve(compileConfig.projectDir)); - if ( - !fs.existsSync( - path.join(compileConfig.projectDir, "node_modules", "@dataform", "core", "bundle.js") - ) - ) { + const coreBundlePath = path.join( + compileConfig.projectDir, "node_modules", "@dataform", "core", "bundle.js" + ); + if (!fs.existsSync(coreBundlePath)) { throw new Error( "Could not find a recent installed version of @dataform/core in the project. Check that " + "either `dataformCoreVersion` is specified in `workflow_settings.yaml`, or " + @@ -21,9 +20,13 @@ export function compile(compileConfig: dataform.ICompileConfig) { "`dataform install`." ); } + const vmIndexFileName = path.resolve(path.join(compileConfig.projectDir, "index.js")); - // First retrieve a compiler function for vm2 to process files. + // Retrieve compiler and version from the resolved @dataform/core. Going + // through Node's resolver inside the vm covers every install layout + // (package.json, workflow_settings.yaml, JiT) and matches what the user's + // code will see. require() caches the bundle so the second call is free. const indexGeneratorVm = new NodeVM({ wrapper: "none", require: { @@ -37,10 +40,47 @@ export function compile(compileConfig: dataform.ICompileConfig) { 'return require("@dataform/core").compiler', vmIndexFileName ); + const dataformCoreVersion: string = indexGeneratorVm.run( + 'return require("@dataform/core").version || "0.0.0"', + vmIndexFileName + ); + + const cliVersion = readCliVersion(); + const cliParsed = semver.parse(cliVersion); + // cliParsed is null for unparseable strings, and "0.0.0" is the sentinel + // returned when package.json can't be read (unbundled local dev). In both + // cases skip the check rather than reject every real Core install. + if (cliParsed && cliVersion !== "0.0.0") { + const minCoreVersion = `${cliParsed.major}.${cliParsed.minor}.0`; + if ( + semver.major(dataformCoreVersion) !== cliParsed.major || + semver.lt(dataformCoreVersion, minCoreVersion) + ) { + throw new Error( + `@dataform/core ${dataformCoreVersion} is not compatible with @dataform/cli ` + + `${cliVersion}. The CLI requires @dataform/core >= ${minCoreVersion} ` + + `(matching major.minor). Set \`dataformCoreVersion: ${cliVersion}\` in ` + + `workflow_settings.yaml (or pin @dataform/core in package.json), then run ` + + `\`dataform install\`.` + ); + } + } + const needsCallerFileShim = semver.lt(dataformCoreVersion, "3.0.57"); + + // vm2 strips file paths from V8 CallSite objects inside the sandbox, so + // getCallerFile() in @dataform/core needs a fallback. Track the currently + // executing file via a host-side stack exposed through sandbox helpers, and + // expose it as a getter on `global.__dataform_current_file`. + const fileStack: string[] = []; // Then use vm2's native compiler integration to apply the compiler to files. const userCodeVm = new NodeVM({ wrapper: "none", + sandbox: { + __df_enter: (p: string) => { fileStack.push(p); }, + __df_exit: () => { fileStack.pop(); }, + __df_current: () => fileStack.length > 0 ? fileStack[fileStack.length - 1] : null + }, require: { builtin: ["path"], context: "sandbox", @@ -50,33 +90,23 @@ export function compile(compileConfig: dataform.ICompileConfig) { path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName) }, sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml"], - // vm2 3.11.3 strips file paths from V8 CallSite objects inside the sandbox, - // which breaks getCallerFile() in @dataform/core. Wrap each compiled module so - // the current file path is exposed via a global, used as a fallback when the - // stack-trace path is unavailable. The try/finally restores the previous value - // to keep nested requires (macros) consistent. compiler: (code, filePath) => { - const compiledCode = compiler(code, filePath); + let source = code; + if (needsCallerFileShim && filePath === coreBundlePath) { + source = patchOldCoreCallerFile(source); + } + const compiledCode = compiler(source, filePath); return ` - var __old_file = global.__dataform_current_file; - global.__dataform_current_file = ${JSON.stringify(filePath)}; + __df_enter(${JSON.stringify(filePath)}); try { ${compiledCode} } finally { - global.__dataform_current_file = __old_file; + __df_exit(); } `; } }); - const dataformCoreVersion: string = userCodeVm.run( - 'return require("@dataform/core").version || "0.0.0"', - vmIndexFileName - ); - if (semver.lt(dataformCoreVersion, "3.0.0-alpha.0")) { - throw new Error("@dataform/core ^3.0.0 required."); - } - const hasWorkflowSettingsYaml = fs.existsSync( path.join(compileConfig.projectDir, "workflow_settings.yaml") ); @@ -86,6 +116,10 @@ export function compile(compileConfig: dataform.ICompileConfig) { return userCodeVm.run( ` + Object.defineProperty(global, '__dataform_current_file', { + configurable: true, + get: function() { return __df_current(); } + }); ${hasWorkflowSettingsYaml ? 'global.workflowSettingsYaml = require("./workflow_settings.yaml");' : ''} @@ -124,6 +158,34 @@ if (require.main === module) { listenForCompileRequest(); } +// Reads the CLI's own version from the package.json baked next to the bundle +// by pkg_json(version = DF_VERSION). Returns "0.0.0" when unreadable. +function readCliVersion(): string { + try { + const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, "package.json"), "utf8") + ); + return pkg.version || "0.0.0"; + } catch { + return "0.0.0"; + } +} + +// @dataform/core <= 3.0.56 has no `global.__dataform_current_file` fallback in +// getCallerFile(), so paired with CLI >= 3.0.57 (which uses vm2 with path +// stripping) every action fails with "Unable to find valid caller file". +// Backport the fallback by rewriting the bundle text at load time. Gated on +// version so we never touch newer core bundles whose layout differs. +const OLD_CORE_THROW = + 'if(!t)throw new Error("Unable to find valid caller file; please report this issue.")'; +const OLD_CORE_WITH_FALLBACK = + 'if(!t){if(global.__dataform_current_file){t=global.__dataform_current_file}' + + 'else{throw new Error("Unable to find valid caller file; please report this issue.")}}'; + +function patchOldCoreCallerFile(source: string): string { + return source.replace(OLD_CORE_THROW, OLD_CORE_WITH_FALLBACK); +} + /** * @returns a base64 encoded @see {@link dataform.CoreExecutionRequest} proto. */ diff --git a/testing/run_core.ts b/testing/run_core.ts index a5f23e424..2cbb175e8 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -76,12 +76,21 @@ export function runMainInVm( fs.copySync(`${process.cwd()}/core/node_modules`, `${projectDir}/node_modules`); const compiler = compile as CompilerFunction; + // See cli/vm/compile.ts for why we use a host-side stack + enter/exit helpers + // instead of writing to `global.__dataform_current_file` inside every module. + const fileStack: string[] = []; + // Then use vm2's native compiler integration to apply the compiler to files. const nodeVm = new NodeVM({ // Inheriting the console makes console.logs show when tests are running, which is useful for // debugging. console: "inherit", wrapper: "none", + sandbox: { + __df_enter: (p: string) => { fileStack.push(p); }, + __df_exit: () => { fileStack.pop(); }, + __df_current: () => fileStack.length > 0 ? fileStack[fileStack.length - 1] : null + }, require: { builtin: ["path"], context: "sandbox", @@ -94,12 +103,11 @@ export function runMainInVm( compiler: (code, filePath) => { const compiledCode = compiler(code, filePath); return ` - var __old_file = global.__dataform_current_file; - global.__dataform_current_file = ${JSON.stringify(filePath)}; + __df_enter(${JSON.stringify(filePath)}); try { ${compiledCode} } finally { - global.__dataform_current_file = __old_file; + __df_exit(); } `; } @@ -109,6 +117,10 @@ export function runMainInVm( const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); const encodedCoreExecutionResponse = nodeVm.run( ` + Object.defineProperty(global, '__dataform_current_file', { + configurable: true, + get: function() { return __df_current(); } + }); global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed run_core:", e); } })(); global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })(); return require("@dataform/core").main("${encodedCoreExecutionRequest}") From 00b33a31111d2a7b11378159c5eae7b1a5c62156 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:49:37 +0000 Subject: [PATCH 14/42] Raise default compile timeout to 60s and improve timeout error message (#2192) Default compilation timeout was 30s. Bump to 60s so larger projects don't hit it on a single compile pass. Extends the "Compilation timed out" error to point at --timeout with concrete examples (--timeout=2m, --timeout=1h) so users can self-mitigate without reading docs. --- cli/api/commands/base_worker.ts | 6 +++++- cli/api/utils/constants.ts | 2 +- tests/api/projects.spec.ts | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cli/api/commands/base_worker.ts b/cli/api/commands/base_worker.ts index e2be64b38..2584f722e 100644 --- a/cli/api/commands/base_worker.ts +++ b/cli/api/commands/base_worker.ts @@ -29,7 +29,11 @@ export abstract class BaseWorker { const timeout = setTimeout(() => { terminate(() => - reject(new Error(`Compilation timed out after ${timeoutMillis / 1000} seconds`)) + reject(new Error( + `Compilation timed out after ${timeoutMillis / 1000} seconds. ` + + `To allow more time, re-run with a longer --timeout ` + + `(e.g. --timeout=2m, --timeout=1h).` + )) ); }, timeoutMillis); diff --git a/cli/api/utils/constants.ts b/cli/api/utils/constants.ts index c4049a9df..f7a82675f 100644 --- a/cli/api/utils/constants.ts +++ b/cli/api/utils/constants.ts @@ -1 +1 @@ -export const DEFAULT_COMPILATION_TIMEOUT_MILLIS = 30000; +export const DEFAULT_COMPILATION_TIMEOUT_MILLIS = 60000; diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts index 7bff903d7..c26debef2 100644 --- a/tests/api/projects.spec.ts +++ b/tests/api/projects.spec.ts @@ -700,7 +700,10 @@ suite("examples", () => { }); fail("Compilation timeout Error expected."); } catch (e) { - expect(e.message).to.equal("Compilation timed out after 1 seconds"); + expect(e.message).to.equal( + "Compilation timed out after 1 seconds. To allow more time, " + + "re-run with a longer --timeout (e.g. --timeout=2m, --timeout=1h)." + ); } }); From c8f0b29903f4bfc83daedda157ea2589398c6db7 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:11:53 +0000 Subject: [PATCH 15/42] Bump DF_VERSION from 3.0.58 to 3.0.59 (#2194) --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index ecee4ae76..37336d889 100644 --- a/version.bzl +++ b/version.bzl @@ -1 +1 @@ -DF_VERSION = "3.0.58" +DF_VERSION = "3.0.59" From 5ab292fe9e18ae7a5d3e8a392433b65d2f3c3eda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:40:08 +0200 Subject: [PATCH 16/42] Bump vm2 from 3.11.3 to 3.11.4 (#2188) Bumps [vm2](https://github.com/patriksimek/vm2) from 3.11.3 to 3.11.4. - [Release notes](https://github.com/patriksimek/vm2/releases) - [Changelog](https://github.com/patriksimek/vm2/blob/main/CHANGELOG.md) - [Commits](https://github.com/patriksimek/vm2/compare/v3.11.3...v3.11.4) --- updated-dependencies: - dependency-name: vm2 dependency-version: 3.11.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 44b58b9a0..907782acb 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "uglify-js": "^3.7.7", "untildify": "^4.0.0", "url": "^0.11.0", - "vm2": "3.11.3", + "vm2": "3.11.4", "vsce": "^1.79.5", "vscode-jsonrpc": "^5.0.1", "vscode-languageclient": "^6.1.3", diff --git a/yarn.lock b/yarn.lock index 5b0a95dda..188629d27 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4437,10 +4437,10 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -vm2@3.11.3: - version "3.11.3" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.3.tgz#5323018a93ae2862c9d52b07b658ebb8d74903f0" - integrity sha512-DO1TTKuOc+veL11VNOvJwRab80mghFKE40Av3bl6pdXs11bdiDMuR73owy+dS2EsTZEvRUeBkkBuDVRjV/RgEw== +vm2@3.11.4: + version "3.11.4" + resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.4.tgz#e68ca77fb420bbbd9dc8d25cf54718067df5949b" + integrity sha512-3assDRyB+A9/5rjECD2e2FAlezOfzCqejZy0L36zij4LgzoWZRA3a26q3PZnl6o5PmNjrTBRbAWACpdK/Y3qOQ== dependencies: acorn "^8.15.0" acorn-walk "^8.3.4" From 97a4c229cc07bf6846fc5f9cb519e2c58ec6a196 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Fri, 5 Jun 2026 16:04:04 +0200 Subject: [PATCH 17/42] Extract BigQueryClientProvider for injectable BQ clients (#2181) * Extract BigQueryClientProvider for injectable BQ clients * Address review: collapse BigQueryClientProvider to a function type --- cli/api/dbadapters/bigquery.ts | 62 +++++++++++++---------- cli/api/dbadapters/bigquery_test.ts | 77 +++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index 178c0c7e1..80dfe39d0 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -37,19 +37,44 @@ export interface IBigQueryExecutionOptions { reservation?: string; } +export type BigQueryClientProvider = (projectId?: string) => BigQuery; + +export function createBigQueryClientProvider( + credentials: dataform.IBigQuery +): BigQueryClientProvider { + const clients = new Map(); + return (projectId?: string) => { + projectId = projectId || credentials.projectId; + if (!clients.has(projectId)) { + clients.set( + projectId, + new BigQuery({ + projectId, + scopes: EXTRA_GOOGLE_SCOPES, + location: credentials.location, + credentials: credentials.credentials && JSON.parse(credentials.credentials) + }) + ); + } + return clients.get(projectId); + }; +} + export class BigQueryDbAdapter implements IDbAdapter { private bigQueryCredentials: dataform.IBigQuery; private pool: PromisePoolExecutor; - - private readonly clients = new Map(); - private readonly bigqueryClient?: BigQuery; + private clientProvider: BigQueryClientProvider; constructor( credentials: dataform.IBigQuery, - options?: { concurrencyLimit?: number; bigqueryClient?: BigQuery } + options?: { + concurrencyLimit?: number; + clientProvider?: BigQueryClientProvider; + } ) { this.bigQueryCredentials = credentials; - this.bigqueryClient = options?.bigqueryClient; + this.clientProvider = options?.clientProvider || createBigQueryClientProvider(credentials); + // Bigquery allows 50 concurrent queries, and a rate limit of 100/user/second by default. // These limits should be safely low enough for most projects. this.pool = new PromisePoolExecutor({ @@ -174,7 +199,7 @@ export class BigQueryDbAdapter implements IDbAdapter { datasetIds.map(async datasetId => { const [tables] = await this.getClient(database) .dataset(datasetId) - .getTables(); + .getTables({ autoPaginate: true, maxResults: 1000 }); await Promise.all( tables.map(async table => { const metadata = await this.table({ @@ -272,14 +297,14 @@ export class BigQueryDbAdapter implements IDbAdapter { } public async schemas(database: string): Promise { - const data = await this.getClient(database).getDatasets(); + const data = await this.getClient(database).getDatasets({ autoPaginate: true, maxResults: 1000 }); return data[0].map(dataset => dataset.id); } public async createSchema(database: string, schema: string): Promise { await this.execute( `create schema if not exists \`${database || this.bigQueryCredentials.projectId}.${schema}\``, - { bigquery: { location: this.bigQueryCredentials.location } } + { bigquery: { location: this.bigQueryCredentials.location || "US" } } ); } @@ -320,23 +345,7 @@ export class BigQueryDbAdapter implements IDbAdapter { } private getClient(projectId?: string) { - if (this.bigqueryClient) { - return this.bigqueryClient; - } - projectId = projectId || this.bigQueryCredentials.projectId; - if (!this.clients.has(projectId)) { - this.clients.set( - projectId, - new BigQuery({ - projectId, - scopes: EXTRA_GOOGLE_SCOPES, - location: this.bigQueryCredentials.location, - credentials: - this.bigQueryCredentials.credentials && JSON.parse(this.bigQueryCredentials.credentials) - }) - ); - } - return this.clients.get(projectId); + return this.clientProvider(projectId); } private async runQuery( @@ -544,11 +553,14 @@ function convertFieldType(type: string) { case "INT64": return dataform.Field.Primitive.INTEGER; case "NUMERIC": + case "BIGNUMERIC": return dataform.Field.Primitive.NUMERIC; case "BOOL": case "BOOLEAN": return dataform.Field.Primitive.BOOLEAN; case "STRING": + case "JSON": + case "INTERVAL": return dataform.Field.Primitive.STRING; case "DATE": return dataform.Field.Primitive.DATE; diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index ecff5ed3e..6659941e8 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -17,12 +17,14 @@ suite("BigQueryDbAdapter", () => { const projectId = "project1"; const credentials = dataform.BigQuery.create({ projectId, location: "US" }); - const adapter = new BigQueryDbAdapter(credentials, { bigqueryClient: instance(mockBigQuery) }); + const adapter = new BigQueryDbAdapter(credentials, { + clientProvider: () => instance(mockBigQuery) + }); when(mockBigQuery.dataset(schemaName)).thenReturn(instance(mockDataset)); // getTables returns an array where the first element is an array of tables. // Each table object needs an 'id' property. - when(mockDataset.getTables()).thenReturn(Promise.resolve([[{ id: tableName }]] as any)); + when(mockDataset.getTables(anything())).thenReturn(Promise.resolve([[{ id: tableName }]] as any)); when(mockDataset.table(tableName)).thenReturn(instance(mockTable)); when(mockTable.getMetadata()).thenReturn( Promise.resolve([ @@ -54,10 +56,12 @@ suite("BigQueryDbAdapter", () => { const projectId = "project"; const credentials = dataform.BigQuery.create({ projectId, location: "US" }); - const adapter = new BigQueryDbAdapter(credentials, { bigqueryClient: instance(mockBigQuery) }); + const adapter = new BigQueryDbAdapter(credentials, { + clientProvider: () => instance(mockBigQuery) + }); when(mockBigQuery.dataset(schemaName)).thenReturn(instance(mockDataset)); - when(mockDataset.getTables()).thenReturn(Promise.resolve([[{ id: tableName }]] as any)); + when(mockDataset.getTables(anything())).thenReturn(Promise.resolve([[{ id: tableName }]] as any)); when(mockDataset.table(tableName)).thenReturn(instance(mockTable)); when(mockTable.getMetadata()).thenReturn( Promise.resolve([ @@ -70,7 +74,7 @@ suite("BigQueryDbAdapter", () => { ] as any) ); - when(mockBigQuery.getDatasets()).thenReturn(Promise.resolve([[{ id: schemaName }]] as any)); + when(mockBigQuery.getDatasets(anything())).thenReturn(Promise.resolve([[{ id: schemaName }]] as any)); const result = await adapter.tables(projectId); @@ -79,4 +83,67 @@ suite("BigQueryDbAdapter", () => { expect(result[0].target.schema).to.equal(schemaName); expect(result[0].target.name).to.equal(tableName); }); + + test("setMetadata handles action without columns", async () => { + // Partial mock for BigQuery client to avoid real network calls + const mockBigQuery: any = { + dataset: () => ({ + table: () => ({ + getMetadata: () => Promise.resolve([{ schema: { fields: [] } }]), + setMetadata: (metadata: any) => { + expect(metadata.description).to.equal("test"); + return Promise.resolve([]); + } + }) + }) + }; + + const credentials = dataform.BigQuery.create({ projectId: "p", location: "US" }); + const adapter = new BigQueryDbAdapter(credentials, { + concurrencyLimit: 1, + clientProvider: () => mockBigQuery + }); + + const action = dataform.ExecutionAction.create({ + target: { database: "db", schema: "sch", name: "tab" }, + actionDescriptor: { description: "test" } + // columns is missing/null in this action + }); + + // This should not throw "cannot read property 'find' of undefined" + await adapter.setMetadata(action); + }); + + test("setMetadata correctly maps column descriptions", async () => { + const mockBigQuery: any = { + dataset: () => ({ + table: () => ({ + getMetadata: () => Promise.resolve([{ + schema: { + fields: [{ name: "id", type: "INTEGER" }] + } + }]), + setMetadata: (metadata: any) => { + expect(metadata.schema[0].description).to.equal("id desc"); + return Promise.resolve([]); + } + }) + }) + }; + + const credentials = dataform.BigQuery.create({ projectId: "p", location: "US" }); + const adapter = new BigQueryDbAdapter(credentials, { + concurrencyLimit: 1, + clientProvider: () => mockBigQuery + }); + + const action = dataform.ExecutionAction.create({ + target: { database: "db", schema: "sch", name: "tab" }, + actionDescriptor: { + columns: [{ path: ["id"], description: "id desc" }] + } + }); + + await adapter.setMetadata(action); + }); }); From 1730f586f9867422f3b57e9623b53d5643f519f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 00:08:22 +0200 Subject: [PATCH 18/42] Bump protobufjs from 7.5.5 to 7.5.8 (#2172) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.5 to 7.5.8. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/protobufjs-v7.5.8/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.5...protobufjs-v7.5.8) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.5.8 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 907782acb..f102c21c4 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "parse-duration": "^1.0.0", "prettier": "^1.14.2", "promise-pool-executor": "^1.1.1", - "protobufjs": "^7.5.5", + "protobufjs": "^7.5.8", "protobufjs-cli": "^1.0.0", "readline-sync": "^1.4.9", "request": "^2.88.0", diff --git a/yarn.lock b/yarn.lock index 188629d27..e9a1d126a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -238,10 +238,10 @@ resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" integrity "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU= sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" -"@protobufjs/codegen@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" - integrity "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs= sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" +"@protobufjs/codegen@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.5.tgz#d9315ad7cf3f30aac70bda3c068443dc6f143659" + integrity sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g== "@protobufjs/eventemitter@^1.1.0": version "1.1.0" @@ -261,10 +261,10 @@ resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" integrity "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" -"@protobufjs/inquire@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" - integrity "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" +"@protobufjs/inquire@^1.1.0", "@protobufjs/inquire@^1.1.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.2.tgz#ae64fbc014ff44c8bfad03dd4c93cd2d6a4c82db" + integrity sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw== "@protobufjs/path@^1.1.2": version "1.1.2" @@ -276,10 +276,10 @@ resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" integrity "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" -"@protobufjs/utf8@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" - integrity "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" +"@protobufjs/utf8@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.1.tgz#eaee5900122c110a3dbcb728c0597014a2621774" + integrity sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg== "@rollup/plugin-node-resolve@^7.1.3": version "7.1.3" @@ -3397,21 +3397,21 @@ protobufjs-cli@^1.0.0: tmp "^0.2.1" uglify-js "^3.7.7" -protobufjs@6.8.8, protobufjs@^7.0.0, protobufjs@^7.5.5: - version "7.5.5" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.5.tgz#b7089ca4410374c75150baf277353ef76db69f96" - integrity sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg== +protobufjs@6.8.8, protobufjs@^7.0.0, protobufjs@^7.5.8: + version "7.5.8" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.8.tgz#51b153a06da6e47153a1aa6800cb1253bc502436" + integrity sha512-dvpCIeLPbXZS/Ete7yLaO7RenOdken2NHKykBXbsaGxZT0UTltcarBciw+A78SRQs9iMAAVpsYA+l8b1hTePIA== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" - "@protobufjs/codegen" "^2.0.4" + "@protobufjs/codegen" "^2.0.5" "@protobufjs/eventemitter" "^1.1.0" "@protobufjs/fetch" "^1.1.0" "@protobufjs/float" "^1.0.2" - "@protobufjs/inquire" "^1.1.0" + "@protobufjs/inquire" "^1.1.1" "@protobufjs/path" "^1.1.2" "@protobufjs/pool" "^1.1.0" - "@protobufjs/utf8" "^1.1.0" + "@protobufjs/utf8" "^1.1.1" "@types/node" ">=13.7.0" long "^5.0.0" From c6c226c788d986dec7574b864ec1a6333719eb8a Mon Sep 17 00:00:00 2001 From: Nick Nalivayka Date: Mon, 8 Jun 2026 14:52:36 +0000 Subject: [PATCH 19/42] Add go_package option to db_adapter.proto and jit.proto (#2197) --- protos/db_adapter.proto | 2 ++ protos/jit.proto | 2 ++ 2 files changed, 4 insertions(+) diff --git a/protos/db_adapter.proto b/protos/db_adapter.proto index 3760c672a..f254ee321 100644 --- a/protos/db_adapter.proto +++ b/protos/db_adapter.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package dataform; +option go_package = "github.com/dataform-co/dataform/protos/dataform"; + import "core.proto"; // Common structures for Evaluation, execution and compilation. diff --git a/protos/jit.proto b/protos/jit.proto index ca0d0707e..4807c2f28 100644 --- a/protos/jit.proto +++ b/protos/jit.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package dataform; +option go_package = "github.com/dataform-co/dataform/protos/dataform"; + import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; From 320af99905890f3356bc3338c85819e15dac10a8 Mon Sep 17 00:00:00 2001 From: ikholopov-omni <73560138+ikholopov-omni@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:27:50 +0200 Subject: [PATCH 20/42] Migrate to bazel 7.3.2 (#2187) * Migrate to bazel 7.3.2 * Use fixed ref for github action --- .bazelrc | 2 + .bazelversion | 2 +- .github/workflows/test.yaml | 6 +- MODULE.bazel | 45 ++++++ MODULE.bazel.lock | 159 ++++++++++++++++++++ WORKSPACE | 93 ------------ cli/index_test_base.ts | 10 +- cloudbuild-publish.yaml | 2 +- cloudbuild-test.yaml | 2 +- packages/@dataform/core/webpack.config.js | 12 +- packages/sample-extension/webpack.config.js | 12 +- protos/BUILD | 4 + tools/gcloud/extensions.bzl | 8 + tools/ts_library.bzl | 3 + tools/ts_proto_library.bzl | 8 +- 15 files changed, 264 insertions(+), 104 deletions(-) create mode 100644 MODULE.bazel create mode 100644 MODULE.bazel.lock create mode 100644 tools/gcloud/extensions.bzl diff --git a/.bazelrc b/.bazelrc index 5ea7b22f3..ca2c522f0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,3 +1,5 @@ +common --enable_bzlmod + build --test_output=errors --action_env="GTEST_COLOR=1" build --incompatible_py3_is_default=false diff --git a/.bazelversion b/.bazelversion index 8a30e8f94..eab246c06 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.4.0 +7.3.2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2af6b101e..06f507b60 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,13 +12,15 @@ jobs: uses: actions/checkout@v4 - name: Run unit tests - uses: gcr.io/cloud-builders/bazel:5.4.0 + # 7.3.2 + uses: gcr.io/cloud-builders/bazel@sha256:aa6c5ffd0d6d37960715d9c3392ba885e6484322c81c3604a90ccfb15c07c110 entrypoint: bash with: args: ./scripts/run_tests - name: Run integration tests - uses: gcr.io/cloud-builders/bazel:5.4.0 + # 7.3.2 + uses: gcr.io/cloud-builders/bazel@sha256:aa6c5ffd0d6d37960715d9c3392ba885e6484322c81c3604a90ccfb15c07c110 entrypoint: bash with: args: ./scripts/run_integration_tests diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 000000000..859a852b6 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,45 @@ +module( + name = "df", + version = "1.0.0", +) + +# 1. bazel_skylib +bazel_dep(name = "bazel_skylib", version = "1.7.1") + +# 2. com_google_protobuf +bazel_dep(name = "protobuf", version = "27.3", repo_name = "com_google_protobuf") + + +# 4. rules_go +bazel_dep(name = "rules_go", version = "0.49.0", repo_name = "io_bazel_rules_go") + +# 5. bazel_gazelle +bazel_dep(name = "gazelle", version = "0.37.0", repo_name = "bazel_gazelle") + +# Go dependencies via Gazelle's go_deps extension +go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") +go_deps.module( + path = "google.golang.org/grpc", + sum = "h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=", + version = "v1.27.0", +) +go_deps.module( + path = "golang.org/x/net", + sum = "h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs=", + version = "v0.0.0-20191002035440-2ec189313ef0", +) +go_deps.module( + path = "golang.org/x/text", + sum = "h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=", + version = "v0.3.2", +) +use_repo( + go_deps, + "org_golang_google_grpc", + "org_golang_x_net", + "org_golang_x_text", +) + +# 6. Custom gcloud SDK extension +gcloud_sdk_ext = use_extension("//tools/gcloud:extensions.bzl", "gcloud_sdk_extension") +use_repo(gcloud_sdk_ext, "gcloud_sdk") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock new file mode 100644 index 000000000..502363efb --- /dev/null +++ b/MODULE.bazel.lock @@ -0,0 +1,159 @@ +{ + "lockFileVersion": 11, + "registryFileHashes": { + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/source.json": "14892cc698e02ffedf4967546e6bedb7245015906888d3465fcf27c90a26da10", + "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", + "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", + "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", + "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", + "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", + "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", + "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", + "https://bcr.bazel.build/modules/gazelle/0.37.0/source.json": "b3adc10e2394e7f63ea88fb1d622d4894bfe9ec6961c493ae9a887723ab16831", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", + "https://bcr.bazel.build/modules/googletest/1.14.0/source.json": "2478949479000fdd7de9a3d0107ba2c85bb5f961c3ecb1aa448f52549ce310b5", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", + "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", + "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", + "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", + "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", + "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", + "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/27.3/MODULE.bazel": "d94898cbf9d6d25c0edca2521211413506b68a109a6b01776832ed25154d23d7", + "https://bcr.bazel.build/modules/protobuf/27.3/source.json": "d6fdc641a99c600df6eb0fa5b99879ca497dbcf6fd1287372576a83f82dd93b6", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.49.0/MODULE.bazel": "61cfc1ba17123356d1b12b6c50f6e0162b2cc7fd6f51753c12471e973a0f72a5", + "https://bcr.bazel.build/modules/rules_go/0.49.0/source.json": "ab2261ea5e29d29a41c8e5c67896f946ab7855b786d28fe25d74987b84e5e85d", + "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", + "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/source.json": "5abb45cc9beb27b77aec6a65a11855ef2b55d95dfdc358e9f312b78ae0ba32d5", + "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", + "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/0.0.7/source.json": "355cc5737a0f294e560d52b1b7a6492d4fff2caf0bef1a315df5a298fca2d34a", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", + "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/source.json": "de77e10ff0ab16acbf54e6b46eecd37a99c5b290468ea1aee6e95eb1affdaed7", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", + "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014", + "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.3/source.json": "cd53fe968dc8cd98197c052db3db6d82562960c87b61e7a90ee96f8e4e0dda97", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "//tools/gcloud:extensions.bzl%gcloud_sdk_extension": { + "general": { + "bzlTransitiveDigest": "VrQN9AjKy3YKE3oOI90UXljo6ZU+VuyVKhDJ/6fJnk0=", + "usagesDigest": "G9567r7BOo4v/0F+Q6wGrEvu7How52VjLhNqFE/f/yk=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "gcloud_sdk": { + "bzlFile": "@@//tools/gcloud:repository_rules.bzl", + "ruleClassName": "gcloud_sdk", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [] + } + }, + "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { + "general": { + "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=", + "usagesDigest": "aLmqbvowmHkkBPve05yyDNGN7oh7QE9kBADr3QIZTZs=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_config_apple_cc": { + "bzlFile": "@@apple_support~//crosstool:setup.bzl", + "ruleClassName": "_apple_cc_autoconf", + "attributes": {} + }, + "local_config_apple_cc_toolchains": { + "bzlFile": "@@apple_support~//crosstool:setup.bzl", + "ruleClassName": "_apple_cc_autoconf_toolchains", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support~", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@platforms//host:extension.bzl%host_platform": { + "general": { + "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", + "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "host_platform": { + "bzlFile": "@@platforms//host:extension.bzl", + "ruleClassName": "host_platform_repo", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [] + } + } + } +} diff --git a/WORKSPACE b/WORKSPACE index ee73c05ff..9562aef48 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,38 +1,5 @@ -workspace(name = "df") - load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "bazel_skylib", - sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", - ], -) - -load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") - -bazel_skylib_workspace() - -load("@bazel_skylib//lib:versions.bzl", "versions") - -versions.check(minimum_bazel_version = "0.26.0") - -# proto_library rules implicitly depend on @com_google_protobuf//:protoc, -# which is the proto-compiler. -# This statement defines the @com_google_protobuf repo. -http_archive( - name = "com_google_protobuf", - sha256 = "f7042d540c969b00db92e8e1066a9b8099c8379c33f40f360eb9e1d98a36ca26", - strip_prefix = "protobuf-3.21.12", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.21.12.zip"], -) - -load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") - -protobuf_deps() - http_archive( name = "build_bazel_rules_nodejs", sha256 = "e79c08a488cc5ac40981987d862c7320cee8741122a2649e9b08e850b6f20442", @@ -60,63 +27,3 @@ yarn_install( symlink_node_modules = False, yarn_lock = "//:yarn.lock", ) - -# Go/Gazelle requirements/dependencies. -http_archive( - name = "io_bazel_rules_go", - sha256 = "56d8c5a5c91e1af73eca71a6fab2ced959b67c86d12ba37feedb0a2dfea441a6", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip", - ], -) - -http_archive( - name = "bazel_gazelle", - sha256 = "ecba0f04f96b4960a5b250c8e8eeec42281035970aa8852dda73098274d14a1d", - urls = [ - "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz", - ], -) - -load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") - -go_rules_dependencies() - -go_register_toolchains(version = "1.19.3") - -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") - -gazelle_dependencies() - -# Gcloud SDK binaries. -load("//tools/gcloud:repository_rules.bzl", "gcloud_sdk") - -gcloud_sdk( - name = "gcloud_sdk", -) - -# Go dependencies. - -go_repository( - name = "org_golang_google_grpc", - build_file_proto_mode = "disable", - importpath = "google.golang.org/grpc", - sum = "h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=", - version = "v1.27.0", -) - -go_repository( - name = "org_golang_x_net", - importpath = "golang.org/x/net", - sum = "h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs=", - version = "v0.0.0-20191002035440-2ec189313ef0", -) - -go_repository( - name = "org_golang_x_text", - importpath = "golang.org/x/text", - sum = "h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=", - version = "v0.3.2", -) diff --git a/cli/index_test_base.ts b/cli/index_test_base.ts index b47f53fe9..564d7f435 100644 --- a/cli/index_test_base.ts +++ b/cli/index_test_base.ts @@ -1,9 +1,17 @@ // tslint:disable tsr-detect-non-literal-fs-filename +import * as fs from "fs"; import * as path from "path"; export const DEFAULT_DATABASE = "dataform-open-source"; export const DEFAULT_LOCATION = "US"; export const DEFAULT_RESERVATION = "projects/dataform-open-source/locations/us/reservations/dataform-test"; -export const CREDENTIALS_PATH = path.resolve(process.env.RUNFILES, "df/test_credentials/bigquery.json"); + +const runfilesDir = process.env.RUNFILES; +let workspaceName = "df"; +if (!fs.existsSync(path.resolve(runfilesDir, "df"))) { + workspaceName = "_main"; +} + +export const CREDENTIALS_PATH = path.resolve(runfilesDir, workspaceName, "test_credentials/bigquery.json"); export const cliEntryPointPath = "cli/node_modules/@dataform/cli/bundle.js"; diff --git a/cloudbuild-publish.yaml b/cloudbuild-publish.yaml index eaea1f75f..f6e86763a 100644 --- a/cloudbuild-publish.yaml +++ b/cloudbuild-publish.yaml @@ -5,7 +5,7 @@ steps: gcloud secrets versions access latest --secret=github-token-access --format='get(payload.data)' | tr '_-' '/+' | base64 -d > ~/token.txt REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN=$NPM_TOKEN ./scripts/create_npmrc secretEnv: ['NPM_TOKEN'] - - name: gcr.io/cloud-builders/bazel:5.4.0 + - name: gcr.io/cloud-builders/bazel:7.3.2 script: | #!/usr/bin/env bash ./scripts/publish diff --git a/cloudbuild-test.yaml b/cloudbuild-test.yaml index 09207ec08..b29e36fc2 100644 --- a/cloudbuild-test.yaml +++ b/cloudbuild-test.yaml @@ -3,7 +3,7 @@ steps: script: | #!/usr/bin/env bash REPO_TOKEN="$(gcloud auth print-access-token)" NPM_TOKEN="test_only" ./scripts/create_npmrc - - name: gcr.io/cloud-builders/bazel:5.4.0 + - name: gcr.io/cloud-builders/bazel:7.3.2 script: | #!/usr/bin/env bash ./scripts/run_tests_on_cloudbuild diff --git a/packages/@dataform/core/webpack.config.js b/packages/@dataform/core/webpack.config.js index 2e389a042..5c2328c4d 100644 --- a/packages/@dataform/core/webpack.config.js +++ b/packages/@dataform/core/webpack.config.js @@ -1,11 +1,19 @@ const path = require("path"); const webpack = require("webpack"); +const fs = require("fs"); + module.exports = (env, argv) => { + const runfilesDir = process.env.RUNFILES; + let workspaceName = "df"; + if (!fs.existsSync(path.resolve(runfilesDir, "df"))) { + workspaceName = "_main"; + } + const config = { mode: argv.mode || "development", target: 'node', - entry: [path.resolve(process.env.RUNFILES, "df/packages/@dataform/core/index")], + entry: [path.resolve(runfilesDir, workspaceName, "packages/@dataform/core/index")], output: { libraryTarget: "commonjs-module", }, @@ -18,7 +26,7 @@ module.exports = (env, argv) => { resolve: { extensions: [".ts", ".js", ".json"], alias: { - df: path.resolve(process.env.RUNFILES, "df") + df: path.resolve(runfilesDir, workspaceName) } }, plugins: [ diff --git a/packages/sample-extension/webpack.config.js b/packages/sample-extension/webpack.config.js index c659abfba..da7ea03bd 100644 --- a/packages/sample-extension/webpack.config.js +++ b/packages/sample-extension/webpack.config.js @@ -1,11 +1,19 @@ const path = require("path"); const webpack = require("webpack"); +const fs = require("fs"); + module.exports = (env, argv) => { + const runfilesDir = process.env.RUNFILES; + let workspaceName = "df"; + if (!fs.existsSync(path.resolve(runfilesDir, "df"))) { + workspaceName = "_main"; + } + const config = { mode: argv.mode || "development", target: 'node', - entry: [path.resolve(process.env.RUNFILES, "df/packages/sample-extension/index")], + entry: [path.resolve(runfilesDir, workspaceName, "packages/sample-extension/index")], output: { libraryTarget: "commonjs-module", }, @@ -18,7 +26,7 @@ module.exports = (env, argv) => { resolve: { extensions: [".ts", ".js", ".json"], alias: { - df: path.resolve(process.env.RUNFILES, "df") + df: path.resolve(runfilesDir, workspaceName) } }, plugins: [ diff --git a/protos/BUILD b/protos/BUILD index 4466c8878..cf9cb3b91 100644 --- a/protos/BUILD +++ b/protos/BUILD @@ -15,14 +15,18 @@ proto_library( "profiles.proto", "extension.proto", ], + strip_import_prefix = "/protos", deps = [ "@com_google_protobuf//:empty_proto", "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", ], ) ts_proto_library( name = "ts", + module_name = "df/protos/ts", + module_root = "ts", deps = [ ":dataform_proto", ], diff --git a/tools/gcloud/extensions.bzl b/tools/gcloud/extensions.bzl new file mode 100644 index 000000000..926fffcdc --- /dev/null +++ b/tools/gcloud/extensions.bzl @@ -0,0 +1,8 @@ +load("//tools/gcloud:repository_rules.bzl", "gcloud_sdk") + +def _gcloud_sdk_extension_impl(ctx): + gcloud_sdk(name = "gcloud_sdk") + +gcloud_sdk_extension = module_extension( + implementation = _gcloud_sdk_extension_impl, +) diff --git a/tools/ts_library.bzl b/tools/ts_library.bzl index fb7ff2b8f..7a6c43211 100644 --- a/tools/ts_library.bzl +++ b/tools/ts_library.bzl @@ -1,6 +1,9 @@ load("@npm//@bazel/typescript:index.bzl", native_ts_library = "ts_library") def ts_library(**kwargs): + if "module_name" not in kwargs: + package = native.package_name() + kwargs["module_name"] = "df/" + package if package else "df" native_ts_library( devmode_target = "es2017", prodmode_target = "es2017", diff --git a/tools/ts_proto_library.bzl b/tools/ts_proto_library.bzl index 3caf17612..69ec2ec37 100644 --- a/tools/ts_proto_library.bzl +++ b/tools/ts_proto_library.bzl @@ -51,13 +51,17 @@ def _ts_proto_library(ctx): output_name = ctx.attr.output_name or ctx.label.name + workspace_name = ctx.workspace_name + if workspace_name == "_main" or not workspace_name: + workspace_name = "df" + js_es5 = _run_pbjs( ctx.actions, ctx.executable, output_name, sources, amd_name = "/".join([p for p in [ - ctx.workspace_name, + workspace_name, ctx.label.package, ] if p]), ) @@ -109,6 +113,8 @@ def _ts_proto_library(ctx): ts_proto_library = rule( implementation = _ts_proto_library, attrs = { + "module_name": attr.string(), + "module_root": attr.string(), "output_name": attr.string( doc = """Name of the resulting module, which you will import from. If not specified, the name will match the target's name.""", From 9d9f48be235564627882ff90e9eaa2a727c10871 Mon Sep 17 00:00:00 2001 From: ikholopov-omni <73560138+ikholopov-omni@users.noreply.github.com> Date: Wed, 17 Jun 2026 08:15:04 +0200 Subject: [PATCH 21/42] Update gcloud to 572.0.0 (#2207) KMS has an org-policy enabled, requiring the usage of client certificate aware endpoints when performing the decryption of credentials. Updating gcloud, so that it uses an updated compatible endpoint. --- MODULE.bazel.lock | 2 +- tools/gcloud/repository_rules.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 502363efb..b14d15b88 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -95,7 +95,7 @@ "moduleExtensions": { "//tools/gcloud:extensions.bzl%gcloud_sdk_extension": { "general": { - "bzlTransitiveDigest": "VrQN9AjKy3YKE3oOI90UXljo6ZU+VuyVKhDJ/6fJnk0=", + "bzlTransitiveDigest": "kvgSakWSAojqjpVE9x4s5Cr2/DcTP4DJDorH/EkVOnU=", "usagesDigest": "G9567r7BOo4v/0F+Q6wGrEvu7How52VjLhNqFE/f/yk=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/tools/gcloud/repository_rules.bzl b/tools/gcloud/repository_rules.bzl index ff714286f..eaf06b466 100644 --- a/tools/gcloud/repository_rules.bzl +++ b/tools/gcloud/repository_rules.bzl @@ -14,6 +14,6 @@ def _gcloud_sdk_impl(ctx): gcloud_sdk = repository_rule( implementation = _gcloud_sdk_impl, attrs = { - "version": attr.string(default = "412.0.0"), + "version": attr.string(default = "572.0.0"), }, ) From c4af15ea39f5254040d59510b83a3e864c0c1edc Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:28:52 +0000 Subject: [PATCH 22/42] Add metadata field to AssertionConfig (#2208) Mirrors TableConfig.metadata / ViewConfig.metadata so callers can attach key/value data that lands on Assertion.action_descriptor.metadata.extra_properties. --- core/actions/assertion.ts | 6 ++++++ core/actions/assertion_test.ts | 18 +++++++++++++++++- protos/configs.proto | 3 +++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts index 34cfa48b8..954293a1e 100644 --- a/core/actions/assertion.ts +++ b/core/actions/assertion.ts @@ -161,6 +161,12 @@ export class Assertion extends ActionBuilder { } this.proto.actionDescriptor.reservation = config.reservation; } + if (config.metadata) { + if (!this.proto.actionDescriptor) { + this.proto.actionDescriptor = {}; + } + this.proto.actionDescriptor.metadata = config.metadata; + } return this; } diff --git a/core/actions/assertion_test.ts b/core/actions/assertion_test.ts index 02ec6e637..9deeedad8 100644 --- a/core/actions/assertion_test.ts +++ b/core/actions/assertion_test.ts @@ -104,6 +104,14 @@ actions: description: "description", hermetic: true, dependOnDependencyAssertions: true, + metadata: { + overview: "assertion overview", + extraProperties: { + fields: { + priority: { stringValue: "high" } + } + } + } }`; [ { @@ -147,7 +155,15 @@ SELECT 1` name: "name" }, actionDescriptor: { - description: "description" + description: "description", + metadata: { + overview: "assertion overview", + extraProperties: { + fields: { + priority: { stringValue: "high" } + } + } + } }, dependencyTargets: [ { diff --git a/protos/configs.proto b/protos/configs.proto index ad8668c96..a0aca3f39 100644 --- a/protos/configs.proto +++ b/protos/configs.proto @@ -528,6 +528,9 @@ message ActionConfig { // If unset, the value from workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies. // Dataform CLI only (GCP Dataform support pending). string reservation = 11; + + // Metadata for this assertion. + Metadata metadata = 12; } message OperationConfig { From 5afdeb4266c71e9915298e80533504adec30f525 Mon Sep 17 00:00:00 2001 From: rafal-hawrylak <33635872+rafal-hawrylak@users.noreply.github.com> Date: Tue, 23 Jun 2026 08:13:52 +0000 Subject: [PATCH 23/42] Bump DF_VERSION from 3.0.59 to 3.0.60 (#2209) --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 37336d889..9c2cb19b3 100644 --- a/version.bzl +++ b/version.bzl @@ -1 +1 @@ -DF_VERSION = "3.0.59" +DF_VERSION = "3.0.60" From 45a6aba65de2fc927118e9d7b1e8bec330e0fbe9 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Tue, 23 Jun 2026 21:41:12 +0100 Subject: [PATCH 24/42] Ignore trailing positional arguments in the global flag parser (#2199) The flag parser in common/flags/index.ts runs in a static initialiser at module load and parses process.argv. Once it had seen a --flag, any later token that was neither another flag nor a flag value caused it to throw "Arg neither flag name nor flag value", crashing the entire CLI before yargs ran. yargs accepts positional arguments after flags (for example `dataform run --some-flag value my_project`), so this lightweight parser was rejecting valid argument orderings. Ignore any token that is neither a flag nor a flag value instead of throwing, exactly as already happened for positional arguments before the first flag; yargs remains responsible for parsing positionals. Extract the parsing loop into an exported parseArgv function so it can be unit-tested in isolation, and add common/flags/index_test.ts covering the regression and related orderings. Fixes #2198. --- common/flags/BUILD | 22 ++++++++++++ common/flags/index.ts | 71 +++++++++++++++++++------------------- common/flags/index_test.ts | 57 ++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 common/flags/index_test.ts diff --git a/common/flags/BUILD b/common/flags/BUILD index 3d6514c38..778c0acc3 100644 --- a/common/flags/BUILD +++ b/common/flags/BUILD @@ -1,16 +1,38 @@ package(default_visibility = ["//visibility:public"]) load("//tools:ts_library.bzl", "ts_library") +load("@df//testing:index.bzl", "ts_test_suite") ts_library( name = "flags", srcs = glob( ["*.ts"], + exclude = ["*_test.ts", "*.spec.ts"], ), deps = [ "//:modules-fix", "@npm//@types/long", "@npm//@types/node", + "@npm//long", + ], +) + +ts_test_suite( + name = "tests", + srcs = glob(["*_test.ts"]), + data = [ + "@npm//source-map-support", + ], + templated_args = [ + "--node_options=--require=source-map-support/register", + "--bazel_patch_module_resolver", + ], + deps = [ + ":flags", + "@df//testing", + "@npm//@types/chai", + "@npm//@types/node", + "@npm//chai", ], ) diff --git a/common/flags/index.ts b/common/flags/index.ts index 6173da668..d9c72f3e0 100644 --- a/common/flags/index.ts +++ b/common/flags/index.ts @@ -39,47 +39,48 @@ export class Flags { Flags.parsedArgv[flagName] = value; } - private static readonly parsedArgv = (() => { - const parsedArgv: { [flagName: string]: string } = {}; - - const splitArgv = []; - for (let arg of process.argv) { - // TODO: This is a temporary hack to be backwards-compatible with yargs behaviour, where - // to switch off a boolean flag, it requires a 'no-' prefix in front of the flag name. - if (arg.startsWith("--no-")) { - arg = `--${arg.slice(5)}=false`; - } - if (arg.startsWith("--") && arg.includes("=")) { - splitArgv.push(arg.slice(0, arg.indexOf("="))); - splitArgv.push(arg.slice(arg.indexOf("=") + 1)); - } else { - splitArgv.push(arg); - } - } - - let flagsStarted = false; - let currentFlagName = ""; - for (const splitArg of splitArgv) { - if (splitArg.startsWith("--")) { - flagsStarted = true; - currentFlagName = splitArg.slice(2); - parsedArgv[currentFlagName] = ""; - } else if (currentFlagName) { - parsedArgv[currentFlagName] = splitArg; - currentFlagName = ""; - } else if (flagsStarted) { - throw new Error(`Arg neither flag name nor flag value: ${splitArg}`); - } - } - - return parsedArgv; - })(); + private static readonly parsedArgv = parseArgv(process.argv); private static invalidFlagValueError(flagName: string) { return new Error(`Invalid flag value: ${Flags.getRawFlagValue(flagName)} [${flagName}]`); } } +// Parses an argv array into a map of flag name to flag value. Any token that is neither a flag +// nor a flag value (for example the command and its positional arguments) is ignored, as yargs +// is responsible for parsing those. +export function parseArgv(argv: string[]): { [flagName: string]: string } { + const parsedArgv: { [flagName: string]: string } = {}; + + const splitArgv = []; + for (let arg of argv) { + // TODO: This is a temporary hack to be backwards-compatible with yargs behaviour, where + // to switch off a boolean flag, it requires a 'no-' prefix in front of the flag name. + if (arg.startsWith("--no-")) { + arg = `--${arg.slice(5)}=false`; + } + if (arg.startsWith("--") && arg.includes("=")) { + splitArgv.push(arg.slice(0, arg.indexOf("="))); + splitArgv.push(arg.slice(arg.indexOf("=") + 1)); + } else { + splitArgv.push(arg); + } + } + + let currentFlagName = ""; + for (const splitArg of splitArgv) { + if (splitArg.startsWith("--")) { + currentFlagName = splitArg.slice(2); + parsedArgv[currentFlagName] = ""; + } else if (currentFlagName) { + parsedArgv[currentFlagName] = splitArg; + currentFlagName = ""; + } + } + + return parsedArgv; +} + export interface IFlag { get(): T; } diff --git a/common/flags/index_test.ts b/common/flags/index_test.ts new file mode 100644 index 000000000..2da49e50b --- /dev/null +++ b/common/flags/index_test.ts @@ -0,0 +1,57 @@ +import { expect } from "chai"; + +import { parseArgv } from "df/common/flags"; +import { suite, test } from "df/testing"; + +suite("flags", () => { + suite("parseArgv", () => { + test("parses a flag followed by its value", () => { + expect(parseArgv(["node", "script", "--foo", "bar"])).deep.equals({ foo: "bar" }); + }); + + test("parses a --flag=value pair", () => { + expect(parseArgv(["node", "script", "--foo=bar"])).deep.equals({ foo: "bar" }); + }); + + test("parses multiple flags", () => { + expect(parseArgv(["node", "script", "--foo", "bar", "--baz", "qux"])).deep.equals({ + foo: "bar", + baz: "qux" + }); + }); + + test("ignores positional arguments before flags", () => { + expect(parseArgv(["node", "script", "run", "my_project", "--foo", "bar"])).deep.equals({ + foo: "bar" + }); + }); + + test("ignores positional arguments after a flag and its value", () => { + // Regression test for https://github.com/dataform-co/dataform/issues/2198: a positional + // argument after a flag used to throw "Arg neither flag name nor flag value", which + // crashed the CLI. + expect(parseArgv(["node", "script", "run", "--foo", "bar", "my_project"])).deep.equals({ + foo: "bar" + }); + }); + + test("ignores positional arguments interleaved with flags", () => { + expect(parseArgv(["node", "script", "--foo", "bar", "pos", "--baz", "qux"])).deep.equals({ + foo: "bar", + baz: "qux" + }); + }); + + test("returns an empty object when no flags are present", () => { + expect(parseArgv(["node", "script", "run", "my_project"])).deep.equals({}); + }); + + test("treats a --no- prefix as setting the flag to false", () => { + expect(parseArgv(["node", "script", "--no-foo"])).deep.equals({ foo: "false" }); + }); + + test("records a value-less trailing flag as an empty string", () => { + expect(parseArgv(["node", "script", "--foo"])).deep.equals({ foo: "" }); + }); + }); +}); From bcd3716aaddbc4d3a1a7a777a9d32ced2ea365c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:18:07 +0200 Subject: [PATCH 25/42] Bump protobufjs-cli from 1.0.0 to 1.3.3 (#2202) * Bump protobufjs-cli from 1.0.0 to 1.3.3 Bumps [protobufjs-cli](https://github.com/protobufjs/protobuf.js) from 1.0.0 to 1.3.3. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-cli-v1.0.0...protobufjs-cli-v1.3.3) --- updated-dependencies: - dependency-name: protobufjs-cli dependency-version: 1.3.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * fix(tools): patch generated proto typings to use ES6 import for 'long' This avoids syntax errors in older rollup-plugin-dts which does not support 'import Long = require("long")' syntax. We patch the generated ts.d.ts file to use 'import Long from "long"' instead. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nick Nalivaika --- package.json | 2 +- tools/ts_proto_library.bzl | 19 ++++--- yarn.lock | 108 +++++++++++++++++++++++++------------ 3 files changed, 87 insertions(+), 42 deletions(-) diff --git a/package.json b/package.json index f102c21c4..371519ba3 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "prettier": "^1.14.2", "promise-pool-executor": "^1.1.1", "protobufjs": "^7.5.8", - "protobufjs-cli": "^1.0.0", + "protobufjs-cli": "^1.3.3", "readline-sync": "^1.4.9", "request": "^2.88.0", "rimraf": "^2.6.2", diff --git a/tools/ts_proto_library.bzl b/tools/ts_proto_library.bzl index 69ec2ec37..c67a2bb45 100644 --- a/tools/ts_proto_library.bzl +++ b/tools/ts_proto_library.bzl @@ -28,16 +28,21 @@ def _run_pbts(actions, executable, js_file): # Reference of arguments: # https://github.com/dcodeIO/ProtoBuf.js/#pbts-for-typescript - args = actions.args() - args.add_all(["--out", ts_file.path]) - args.add(js_file.path) + # + # Run pbts and then patch 'import Long = require("long");' to 'import Long from "long";' + # to avoid syntax errors in older rollup-plugin-dts. + command = "{pbts} --out {out} {input} && sed -i 's/import Long = require(\"long\");/import Long from \"long\";/g' {out}".format( + pbts = executable._pbts.path, + out = ts_file.path, + input = js_file.path + ) - actions.run( - executable = executable._pbts, - progress_message = "Generating typings from %s" % js_file.short_path, + actions.run_shell( inputs = [js_file], outputs = [ts_file], - arguments = [args], + tools = [executable._pbts], + command = command, + progress_message = "Generating typings from %s" % js_file.short_path, ) return ts_file diff --git a/yarn.lock b/yarn.lock index e9a1d126a..c24e8c6e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16,6 +16,16 @@ dependencies: "@babel/highlight" "^7.8.3" +"@babel/helper-string-parser@^7.29.7": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz#7f0871d99824d23137d60f86fcf6130fd5a1b51f" + integrity sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw== + +"@babel/helper-validator-identifier@^7.29.7": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz#bd87084ced0c796ec46bda492de6e83d29e89fc2" + integrity sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg== + "@babel/helper-validator-identifier@^7.9.0": version "7.9.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.5.tgz#90977a8e6fbf6b431a7dc31752eee233bf052d80" @@ -39,11 +49,26 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/parser@^7.20.15": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.29.7.tgz#837b87387cbf5ec5530cb634b3c622f68edb9334" + integrity sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg== + dependencies: + "@babel/types" "^7.29.7" + "@babel/parser@^7.9.4": version "7.13.12" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.12.tgz#ba320059420774394d3b0c0233ba40e4250b81d1" integrity "sha1-ujIAWUIHdDlNOwwCM7pA5CULgdE= sha512-4T7Pb244rxH24yR116LAuJ+adxXXnHhZaLJjegJVKSdoNCe4x1eDBaud5YIcQFcqzsaD5BHvJw5BQ0AZapdCRw==" +"@babel/types@^7.29.7": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.29.7.tgz#8005e31d82712ee7adaef6e23c63b71a62770a92" + integrity sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA== + dependencies: + "@babel/helper-string-parser" "^7.29.7" + "@babel/helper-validator-identifier" "^7.29.7" + "@bazel/hide-bazel-files@1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@bazel/hide-bazel-files/-/hide-bazel-files-1.0.0.tgz#779070dcb5ae121ff6c72d73bf3fb7bf68285d75" @@ -223,6 +248,13 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@jsdoc/salty@^0.2.1": + version "0.2.12" + resolved "https://registry.yarnpkg.com/@jsdoc/salty/-/salty-0.2.12.tgz#6320bb4bd16e98c2da2c666ec97d1a72922cd06a" + integrity sha512-TuB0x50EoAvEX/UEWITd8Mkn3WhiTjSvbTMCLj0BhsQEl5iUzjXdA0bETEVpTk+5TGTLR6QktI9H4hLviVeaAQ== + dependencies: + lodash "^4.18.1" + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" @@ -411,6 +443,11 @@ resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.2.tgz#fd2cd2edbaa7eaac7e7f3c1748b52a19143846c9" integrity "sha1-/SzS7bqn6qx+fzwXSLUqGRQ4Rsk= sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==" +"@types/linkify-it@^5": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-5.0.0.tgz#21413001973106cda1c3a9b91eedd4ccd5469d76" + integrity sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q== + "@types/long@^4.0.0": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" @@ -424,11 +461,24 @@ "@types/linkify-it" "*" "@types/mdurl" "*" +"@types/markdown-it@^14.1.1": + version "14.1.2" + resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-14.1.2.tgz#57f2532a0800067d9b934f3521429a2e8bfb4c61" + integrity sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog== + dependencies: + "@types/linkify-it" "^5" + "@types/mdurl" "^2" + "@types/mdurl@*": version "1.0.2" resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-1.0.2.tgz#e2ce9d83a613bacf284c7be7d491945e39e1f8e9" integrity "sha1-4s6dg6YTus8oTHvn1JGUXjnh+Ok= sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==" +"@types/mdurl@^2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-2.0.0.tgz#d43878b5b20222682163ae6f897b20447233bdfd" + integrity sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg== + "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -1683,16 +1733,11 @@ esrecurse@^4.3.0: dependencies: estraverse "^5.2.0" -estraverse@^4.1.1: +estraverse@^4.1.1, estraverse@^4.2.0: version "4.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== -estraverse@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.2.0.tgz#0dee3fed31fcd469618ce7342099fc1afa0bdb13" - integrity "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM= sha512-VHvyaGnJy+FuGfcfaM7W7OZw4mQiKW73jPHwQXx2VnMSUBajYmytOT5sKEfsBvNPtGX6YDwcrGDz2eocoHg0JA==" - estraverse@^5.1.0, estraverse@^5.2.0: version "5.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" @@ -2625,25 +2670,25 @@ jsdoc@^3.6.11: taffydb "2.6.2" underscore "~1.13.2" -jsdoc@^3.6.3: - version "3.6.10" - resolved "https://registry.yarnpkg.com/jsdoc/-/jsdoc-3.6.10.tgz#dc903c44763b78afa7d94d63da475d20bc224cc4" - integrity "sha1-3JA8RHY7eK+n2U1j2kddILwiTMQ= sha512-IdQ8ppSo5LKZ9o3M+LKIIK8i00DIe5msDvG3G81Km+1dhy0XrOWD0Ji8H61ElgyEj/O9KRLokgKbAM9XX9CJAg==" +jsdoc@^4.0.0: + version "4.0.5" + resolved "https://registry.yarnpkg.com/jsdoc/-/jsdoc-4.0.5.tgz#fbed70e04a3abcf2143dad6b184947682bbc7315" + integrity sha512-P4C6MWP9yIlMiK8nwoZvxN84vb6MsnXcHuy7XzVOvQoCizWX5JFCBsWIIWKXBltpoRZXddUOVQmCTOZt9yDj9g== dependencies: - "@babel/parser" "^7.9.4" - "@types/markdown-it" "^12.2.3" + "@babel/parser" "^7.20.15" + "@jsdoc/salty" "^0.2.1" + "@types/markdown-it" "^14.1.1" bluebird "^3.7.2" catharsis "^0.9.0" escape-string-regexp "^2.0.0" js2xmlparser "^4.0.2" - klaw "^4.0.1" - markdown-it "^12.3.2" - markdown-it-anchor "^8.4.1" + klaw "^3.0.0" + markdown-it "^14.1.0" + markdown-it-anchor "^8.6.7" marked "^4.0.10" mkdirp "^1.0.4" requizzle "^0.2.3" strip-json-comments "^3.1.0" - taffydb "2.6.2" underscore "~1.13.2" json-bigint@^1.0.0: @@ -2771,11 +2816,6 @@ klaw@^3.0.0: dependencies: graceful-fs "^4.1.9" -klaw@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/klaw/-/klaw-4.0.1.tgz#8dc6f5723f05894e8e931b516a8ff15c2976d368" - integrity "sha1-jcb1cj8FiU6OkxtRao/xXCl202g= sha512-pgsE40/SvC7st04AHiISNewaIMUbY5V/K8b21ekiPiFoYs/EYSdsGa+FJArB1d441uq4Q8zZyIxvAzkGNlBdRw==" - leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" @@ -2817,7 +2857,7 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" -lodash@^4.15.0, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.5: +lodash@^4.15.0, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.5, lodash@^4.18.1: version "4.18.1" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.18.1.tgz#ff2b66c1f6326d59513de2407bf881439812771c" integrity sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q== @@ -2879,7 +2919,12 @@ markdown-it-anchor@^8.4.1: resolved "https://registry.yarnpkg.com/markdown-it-anchor/-/markdown-it-anchor-8.6.4.tgz#affb8aa0910a504c114e9fcad53ac3a5b907b0e6" integrity "sha1-r/uKoJEKUEwRTp/K1TrDpbkHsOY= sha512-Ul4YVYZNxMJYALpKtu+ZRdrryYt/GlQ5CK+4l1bp/gWXOG2QWElt6AqF3Mih/wfUKdZbNAZVXGR73/n6U/8img==" -markdown-it@^10.0.0, markdown-it@^12.3.2: +markdown-it-anchor@^8.6.7: + version "8.6.7" + resolved "https://registry.yarnpkg.com/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz#ee6926daf3ad1ed5e4e3968b1740eef1c6399634" + integrity sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA== + +markdown-it@^10.0.0, markdown-it@^12.3.2, markdown-it@^14.1.0: version "12.3.2" resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-12.3.2.tgz#bf92ac92283fe983fe4de8ff8abfb5ad72cd0c90" integrity "sha1-v5Kskig/6YP+Tej/ir+1rXLNDJA= sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==" @@ -3381,17 +3426,17 @@ proto-list@~1.2.1: resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" integrity "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk= sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" -protobufjs-cli@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/protobufjs-cli/-/protobufjs-cli-1.0.0.tgz#62f99de142118f34c7b4ee53c2dc122612d2711c" - integrity "sha1-Yvmd4UIRjzTHtO5TwtwSJhLScRw= sha512-7NZEBrFuuU2ZdzlhmAmvh8fdU7A4OFhzYX9AB7a5vXjopAeiks6ZgUSjOlOO7ItCDJQm3y9RWjk7spUbHc4X0w==" +protobufjs-cli@^1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/protobufjs-cli/-/protobufjs-cli-1.3.3.tgz#72ed54b9b634e9191351d2a4db68363dde76fc4a" + integrity sha512-zwmt6JeStPjeofZbl+QADexAVzP1EgsIAsO7mCfKkW/8oBxHwgTqJ1aD7zDrcCC0Nb+SLWSvCR3RDaKgIO3P8w== dependencies: chalk "^4.0.0" escodegen "^1.13.0" espree "^9.0.0" estraverse "^5.1.0" glob "^8.0.0" - jsdoc "^3.6.3" + jsdoc "^4.0.0" minimist "^1.2.0" semver "^7.1.2" tmp "^0.2.1" @@ -4317,16 +4362,11 @@ uglify-es@^3.3.9: commander "~2.13.0" source-map "~0.6.1" -uglify-js@^3.1.4: +uglify-js@^3.1.4, uglify-js@^3.7.7: version "3.19.3" resolved "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== -uglify-js@^3.7.7: - version "3.16.0" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.16.0.tgz#b778ba0831ca102c1d8ecbdec2d2bdfcc7353190" - integrity "sha1-t3i6CDHKECwdjsvewtK9/Mc1MZA= sha512-FEikl6bR30n0T3amyBh3LoiBdqHRy/f4H80+My34HOesOKyHfOsxAPAxOoqC0JUnC1amnO0IwkYC3sko51caSw==" - underscore@^1.12.1, underscore@~1.13.2: version "1.13.8" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.8.tgz#a93a21186c049dbf0e847496dba72b7bd8c1e92b" From a8cd8f362e88f7750efb98173150c2f92a0ee772 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:03:05 +0200 Subject: [PATCH 26/42] Bump js-yaml from 4.1.1 to 4.2.0 (#2205) Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.1.1 to 4.2.0. - [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/commits) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 4.2.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 371519ba3..cd495f16e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "glob": "13.0.6", "google-sql-syntax-ts": "^1.0.3", "js-beautify": "^1.10.2", - "js-yaml": "^4.1.1", + "js-yaml": "^4.2.0", "jsdoc": "^3.6.11", "json-stable-stringify": "^1.0.1", "long": "^4.0.0", diff --git a/yarn.lock b/yarn.lock index c24e8c6e8..9dfbeceb7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2630,10 +2630,10 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" - integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== +js-yaml@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.2.0.tgz#2bd9e85682dd91bd469afb809d816043b3d49524" + integrity sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw== dependencies: argparse "^2.0.1" From 2276440a95d8073a7138d7eb351724ca88bc236d Mon Sep 17 00:00:00 2001 From: Amman <87317538+AmmanuelT@users.noreply.github.com> Date: Fri, 26 Jun 2026 04:11:50 +0300 Subject: [PATCH 27/42] feat: add getContents() for reading external markdown files into table descriptions pr fix (#2164) * initial attempt at md reader * adding relevant tests and better path resolving for md * cleaner get contents path handling * escaping newlines for md * adding documentation * add check if file in rootdir for get contents and update documentation on getContents to include limits * cleaner and filesystem agnostic check if getContents target is in the root dir * making sure root dir has seperator on check if getcontents file is in dir * removing node module usage * removing redundant normalize and root variable * adding seperator when checking if file is in rootDir --- core/compilers.ts | 8 +++++ core/compilers_test.ts | 42 ++++++++++++++++++++++ core/main.ts | 3 +- core/main_test.ts | 76 +++++++++++++++++++++++++++++++++++++++ core/session.ts | 26 ++++++++++++++ docs/reference/session.md | 29 +++++++++++++++ testing/run_core.ts | 2 +- 7 files changed, 184 insertions(+), 2 deletions(-) diff --git a/core/compilers.ts b/core/compilers.ts index a760153ff..0a00a70c6 100644 --- a/core/compilers.ts +++ b/core/compilers.ts @@ -56,6 +56,14 @@ export function compile(code: string, path: string): string { .replace(/\${/g, "\\${"); return `exports.query = \`${escapedCode}\`;`; } + if (Path.fileExtension(path) === "md") { + const contents = code + .replace(/\r\n/g, "\n") + .replace(/\\/g, "\\\\") + .replace(/`/g, "\\`") + .replace(/\${/g, "\\${"); + return `exports.contents = \`${contents}\`;`; + } return code; } diff --git a/core/compilers_test.ts b/core/compilers_test.ts index f5410e557..03a7424b4 100644 --- a/core/compilers_test.ts +++ b/core/compilers_test.ts @@ -92,5 +92,47 @@ suite("core/compilers", () => { const result = compile(code, path); expect(result).to.equal(code); }); + + test("compiles md to js contents export",() => { + const code = '# this table defines'; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `# this table defines`;"); + + }) + + test("escapes backslashes in md", () => { + const code = "select ''"; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `select ''`;"); + }); + test("escapes newlines in md", () => { + const code = ` +- item1 +- item2`; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `\n- item1\n- item2`;"); + }); + test ("escapes template literals in md", () => { + const code = "select ${foo}"; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `select \\${foo}`;"); + }); + test("escapes backslashes and backticks in md", () => { + const code = "c = '\\'"; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `c = '\\\\'`;"); + }); + test("escapes backticks in md", () => { + const code = "select `a` from `b`"; + const path = "definitions/foo.md"; + const result = compile(code, path); + expect(result).to.equal("exports.contents = `select \\`a\\` from \\`b\\``;"); + }); + }); }); diff --git a/core/main.ts b/core/main.ts index b4ead165a..e0aaf0959 100644 --- a/core/main.ts +++ b/core/main.ts @@ -232,7 +232,8 @@ function dataformCompile(compileRequest: dataform.ICompileExecutionRequest, sess globalAny.notebook = session.notebook.bind(session); globalAny.test = session.test.bind(session); globalAny.jitData = session.jitData.bind(session); - + globalAny.getContents = session.getContents.bind(session); + loadActionConfigs(session, compileRequest.compileConfig.filePaths); // Require all "definitions" files (attaching them to the session). diff --git a/core/main_test.ts b/core/main_test.ts index c23f62f2e..f9c5c8e70 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -1764,7 +1764,83 @@ dataform.jitData("key", {test: () => {}}); ).to.deep.equal(["Unsupported value: () => {}"]); }); }); + suite("markdown in description", () => { + test("markdown contents added to description in sqlx", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/descriptions.md"), + `# This table contains data about` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/table.sqlx"), + `config { + type: "table", + description: getContents('./descriptions.md'), + } + SELECT 1 AS test` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + expect( + result.compile.compiledGraph.tables[0].actionDescriptor.description + ).to.equal(`# This table contains data about`); + + }); + + test("throws error for invalid missing markedown", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/table.sqlx"), + `config { + type: "table", + description: getContents('./nonexistent.md'), + } + SELECT 1 AS test` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors[0].message + ).to.include("nonexistent.md"); + + }); + + test("throws error for file outisde of rootDir", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/table.sqlx"), + `config { + type: "table", + description: getContents('../../description.md'), + } + SELECT 1 AS test` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors[0].message + ).to.include("outside the project directory"); + + }); + }); suite("invalid options", () => { [ { diff --git a/core/session.ts b/core/session.ts index b29bc971f..c078f158b 100644 --- a/core/session.ts +++ b/core/session.ts @@ -16,12 +16,14 @@ import { Test } from "df/core/actions/test"; import { View } from "df/core/actions/view"; import { CompilationSql } from "df/core/compilation_sql"; import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables"; +import * as Path from "df/core/path"; import { targetAsReadableString, targetStringifier } from "df/core/targets"; import * as utils from "df/core/utils"; import { ResolvableMap, toResolvable } from "df/core/utils"; import { version as dataformCoreVersion } from "df/core/version"; import { dataform, google } from "df/protos/ts"; + const DEFAULT_CONFIG = { defaultSchema: "dataform", assertionSchema: "dataform_assertions" @@ -91,6 +93,30 @@ export class Session { return new CompilationSql(this.projectConfig, dataformCoreVersion); } + public getContents(filePath: string): string { + const callerFile = utils.getCallerFile(this.rootDir); + const callerDir = Path.dirName(callerFile); + const resolvedPath = Path.join(callerDir,filePath); + const absolutePath = Path.separator + Path.normalize(Path.join(this.rootDir, resolvedPath)); + const rootDir = this.rootDir.endsWith(Path.separator) ? this.rootDir : this.rootDir + Path.separator; + + if (!absolutePath.startsWith(rootDir)){ + throw new Error(`Cannot read "${filePath}": path resolves outside the project directory.`); + } + + let module: any; + try { + module = utils.nativeRequire(absolutePath); + } catch { + throw new Error(`Cannot read "${filePath}": file not found.`); + } + + if (!module || typeof module.contents !== "string") { + throw new Error (`Cannot read "${filePath}": only .md files are supported.`) + } + return module.contents; + } + public sqlxAction(actionOptions: { // sqlxConfig has type any here because any object can be passed in from the compiler - the // structure of it is verified at later steps. diff --git a/docs/reference/session.md b/docs/reference/session.md index b0da1f981..9c9fe7e16 100644 --- a/docs/reference/session.md +++ b/docs/reference/session.md @@ -19,6 +19,7 @@ folder of a Dataform project. * [assert](_core_session_.session.md#assert) * [declare](_core_session_.session.md#declare) +* [getContents](_core_session_.session.md#getcontents) * [notebook](_core_session_.session.md#notebook) * [operate](_core_session_.session.md#operate) * [publish](_core_session_.session.md#publish) @@ -82,6 +83,34 @@ Name | Type | ___ +### getContents + +▸ **getContents**(`filePath`: string): *string* + +Reads the contents of an external markdown file and returns it as a string. The path is resolved relative to the file that calls `getContents`. + +Available only in the `/definitions` directory. + +**Example:** + +```sqlx +config { + type: "table", + description: getContents('./my_table_description.md') +} +SELECT ... +``` + +**Parameters:** + +Name | Type | Description | +------ | ------ | ------ | +`filePath` | string | Path to the file, relative to the calling file. | + +**Returns:** *string* + +___ + ### notebook ▸ **notebook**(`config`: NotebookConfig): *[Notebook](_core_actions_notebook_.notebook.md)* diff --git a/testing/run_core.ts b/testing/run_core.ts index 2cbb175e8..111262247 100644 --- a/testing/run_core.ts +++ b/testing/run_core.ts @@ -48,7 +48,7 @@ export class WorkflowSettingsTemplates { }); } -const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb"]; +const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb","md"]; export function coreExecutionRequestFromPath( projectDir: string, From 3da6b132ba55d38eda44005a551fa742bcc6e214 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 04:34:39 +0200 Subject: [PATCH 28/42] Bump protobufjs from 7.5.8 to 7.6.3 (#2204) Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.5.8 to 7.6.3. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/protobufjs-v7.6.3/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.5.8...protobufjs-v7.6.3) --- updated-dependencies: - dependency-name: protobufjs dependency-version: 7.6.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 43 +++++++++++++++++++++---------------------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index cd495f16e..d63ebcd57 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "parse-duration": "^1.0.0", "prettier": "^1.14.2", "promise-pool-executor": "^1.1.1", - "protobufjs": "^7.5.8", + "protobufjs": "^7.6.3", "protobufjs-cli": "^1.3.3", "readline-sync": "^1.4.9", "request": "^2.88.0", diff --git a/yarn.lock b/yarn.lock index 9dfbeceb7..41a61ffe5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -275,25 +275,24 @@ resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.5.tgz#d9315ad7cf3f30aac70bda3c068443dc6f143659" integrity sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g== -"@protobufjs/eventemitter@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" - integrity "sha1-NVy8mLr61ZePntCV85diHx0Ga3A= sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" +"@protobufjs/eventemitter@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz#d512cb26c0ae026091ee2c1167f1be6faf5c842a" + integrity sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg== -"@protobufjs/fetch@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" - integrity "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==" +"@protobufjs/fetch@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.1.tgz#4d6fc00c8fb64016a5c81b469d549046350f1065" + integrity sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw== dependencies: "@protobufjs/aspromise" "^1.1.1" - "@protobufjs/inquire" "^1.1.0" "@protobufjs/float@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" integrity "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" -"@protobufjs/inquire@^1.1.0", "@protobufjs/inquire@^1.1.1": +"@protobufjs/inquire@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.2.tgz#ae64fbc014ff44c8bfad03dd4c93cd2d6a4c82db" integrity sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw== @@ -2867,10 +2866,10 @@ long@^4.0.0: resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg= sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" -long@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" - integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== +long@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" + integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== lru-cache@^10.2.0: version "10.4.3" @@ -3442,23 +3441,23 @@ protobufjs-cli@^1.3.3: tmp "^0.2.1" uglify-js "^3.7.7" -protobufjs@6.8.8, protobufjs@^7.0.0, protobufjs@^7.5.8: - version "7.5.8" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.8.tgz#51b153a06da6e47153a1aa6800cb1253bc502436" - integrity sha512-dvpCIeLPbXZS/Ete7yLaO7RenOdken2NHKykBXbsaGxZT0UTltcarBciw+A78SRQs9iMAAVpsYA+l8b1hTePIA== +protobufjs@6.8.8, protobufjs@^7.0.0, protobufjs@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.6.3.tgz#92cc8806db61393e68f6a2eb742c1d9c0cafd672" + integrity sha512-+k0vdJKNdW+Vu+dYe8tZA/VvQb6XKNWexC6URwBFXxNnjLJz9nQJCemGyNgRAWD+B7+nGNc9qMPGwcD7s4nzUw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" "@protobufjs/codegen" "^2.0.5" - "@protobufjs/eventemitter" "^1.1.0" - "@protobufjs/fetch" "^1.1.0" + "@protobufjs/eventemitter" "^1.1.1" + "@protobufjs/fetch" "^1.1.1" "@protobufjs/float" "^1.0.2" - "@protobufjs/inquire" "^1.1.1" + "@protobufjs/inquire" "^1.1.2" "@protobufjs/path" "^1.1.2" "@protobufjs/pool" "^1.1.0" "@protobufjs/utf8" "^1.1.1" "@types/node" ">=13.7.0" - long "^5.0.0" + long "^5.3.2" prr@~1.0.1: version "1.0.1" From 427ebba2faa65c44579d994c8170e2ab720cd00f Mon Sep 17 00:00:00 2001 From: Nick Nalivayka Date: Sat, 27 Jun 2026 12:32:49 +0000 Subject: [PATCH 29/42] feat(gcloud): support client certificate environment in gcloud actions and tests (#2213) - Enable `use_default_shell_env` in `gcloud_secret` rule to pass host environment variables (like `CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE`) to the gcloud tool. - Pass `CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE` to bazel test in `scripts/publish`. --- scripts/publish | 2 +- tools/gcloud/secrets.bzl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/publish b/scripts/publish index 8c585960d..b7397410c 100755 --- a/scripts/publish +++ b/scripts/publish @@ -18,7 +18,7 @@ fi bazel run @nodejs//:yarn config set registry https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/ bazel run //tools/registry-tools:switch_registry -- $(pwd)/yarn.lock https://us-npm.pkg.dev/artifact-foundry-prod/npm-3p-trusted/ bazel run @nodejs//:yarn install -- --frozen-lockfile -bazel test //... --build_tests_only +bazel test //... --build_tests_only --action_env=CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE # After the code is build with Airlock dependencies, we change the registry to public npmjs # to publish our npm package. diff --git a/tools/gcloud/secrets.bzl b/tools/gcloud/secrets.bzl index e456d88b4..862b9fda2 100644 --- a/tools/gcloud/secrets.bzl +++ b/tools/gcloud/secrets.bzl @@ -14,6 +14,7 @@ def _gcloud_secret_impl(ctx): "--location=%s" % ctx.attr.location, "--project=%s" % ctx.attr.project, ], + use_default_shell_env = True, execution_requirements = { "local": "1", }, From e7444f94681378a714e6e05491b74c5ddb48bd47 Mon Sep 17 00:00:00 2001 From: August Cayzer Date: Sat, 27 Jun 2026 13:47:10 +0100 Subject: [PATCH 30/42] Prevent .sqlx files in actions.yaml configs with a clear error (#2189) Putting a .sqlx file under the ``filename`` field of an entry in ``definitions/actions.yaml`` used to fall through to ``nativeRequire``, which then failed with a cryptic error far from the source of the mistake. ``.sqlx`` files are loaded directly from the ``definitions/`` directory by the sqlx compiler, not referenced through ``actions.yaml``. Add an explicit validation step in ``loadActionConfigs`` that runs before each action is constructed. If the action's ``filename`` ends in ``.sqlx`` (case-insensitive), emit a ``compileError`` naming the action type, the offending filename, and pointing at the fix. The action is then skipped so we don't double-up with a downstream ``nativeRequire`` error. The check covers every action type that accepts a ``filename``: ``table``, ``view``, ``incrementalTable``, ``assertion``, ``operation``, ``declaration``, ``notebook`` and ``dataPreparation``. Data preparations are included because a ``.dp.sqlx`` file is also auto-compiled from ``definitions/`` - referencing one from ``actions.yaml`` doesn't resolve its path, ignores its ``config {}`` block and double-registers the action, so it's the same mistake. ``.dp.yaml`` data preparations are unaffected. Closes #1785 --- core/main.ts | 44 ++++++++++++++++ core/main_test.ts | 127 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) diff --git a/core/main.ts b/core/main.ts index e0aaf0959..87df59ee7 100644 --- a/core/main.ts +++ b/core/main.ts @@ -75,6 +75,46 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st return dataform.CoreExecutionResponse.encode(coreExecutionResponse).finish(); } +// Every action type that accepts a `filename` in actions.yaml, paired with its +// statically-typed sub-config. Referencing a .sqlx file (including a +// data-preparation .dp.sqlx file) is unsupported: .sqlx files are compiled +// directly from the definitions/ directory by the sqlx compiler, not through the +// `actions.yaml` config. Without this check a .sqlx filename produces a cryptic +// `nativeRequire` error rather than a useful diagnostic, so we emit a clear +// compilation error up front. See issue #1785. +function actionConfigFilenameIsInvalidSqlx( + session: Session, + actionConfig: dataform.ActionConfig, + actionConfigsPath: string +): boolean { + const filenamesByActionType: Array<[string, string | null | undefined]> = [ + ["table", actionConfig.table?.filename], + ["view", actionConfig.view?.filename], + ["incrementalTable", actionConfig.incrementalTable?.filename], + ["assertion", actionConfig.assertion?.filename], + ["operation", actionConfig.operation?.filename], + ["declaration", actionConfig.declaration?.filename], + ["notebook", actionConfig.notebook?.filename], + ["dataPreparation", actionConfig.dataPreparation?.filename] + ]; + for (const [actionType, filename] of filenamesByActionType) { + if (filename && filename.toLowerCase().endsWith(".sqlx")) { + session.compileError( + new Error( + `Action config "${actionType}" has filename "${filename}", but .sqlx ` + + `files cannot be referenced from actions.yaml. .sqlx files are ` + + `compiled directly from the definitions/ directory. Either use a ` + + `.sql file with the same contents, or remove the actions.yaml ` + + `entry and let Dataform pick up the .sqlx file automatically.` + ), + actionConfigsPath + ); + return true; + } + } + return false; +} + function loadActionConfigs(session: Session, filePaths: string[]) { filePaths .filter( @@ -89,6 +129,10 @@ function loadActionConfigs(session: Session, filePaths: string[]) { actionConfigs.actions.forEach(nonProtoActionConfig => { const actionConfig = dataform.ActionConfig.create(nonProtoActionConfig); + if (actionConfigFilenameIsInvalidSqlx(session, actionConfig, actionConfigsPath)) { + return; + } + if (actionConfig.table) { session.actions.push( new Table( diff --git a/core/main_test.ts b/core/main_test.ts index f9c5c8e70..18d39c2ad 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -994,6 +994,133 @@ actions:` ); }); + suite(`.sqlx filenames in actions.yaml are rejected with a clear error`, () => { + [ + { + actionType: "table", + yamlBody: ` +actions: +- table: + filename: table.sqlx` + }, + { + actionType: "view", + yamlBody: ` +actions: +- view: + filename: view.sqlx` + }, + { + actionType: "incrementalTable", + yamlBody: ` +actions: +- incrementalTable: + filename: incremental.sqlx` + }, + { + actionType: "assertion", + yamlBody: ` +actions: +- assertion: + filename: assertion.sqlx` + }, + { + actionType: "operation", + yamlBody: ` +actions: +- operation: + filename: operation.sqlx` + }, + { + actionType: "declaration", + yamlBody: ` +actions: +- declaration: + name: declaration + filename: declaration.sqlx` + }, + { + actionType: "notebook", + yamlBody: ` +actions: +- notebook: + filename: notebook.sqlx` + } + ].forEach(({ actionType, yamlBody }) => { + test(`for action type "${actionType}"`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/actions.yaml"), yamlBody); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + const errorMessages = result.compile.compiledGraph.graphErrors.compilationErrors.map( + ({ message }) => message + ); + expect(errorMessages).to.have.lengthOf(1); + expect(errorMessages[0]).to.include(`Action config "${actionType}" has filename`); + expect(errorMessages[0]).to.include( + ".sqlx files cannot be referenced from actions.yaml" + ); + }); + }); + + test(`is case-insensitive`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- table: + filename: table.SQLX` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors.map(({ message }) => message) + ).to.have.lengthOf(1); + }); + + test(`for data preparations referencing a .dp.sqlx file`, () => { + // .dp.sqlx data preparations are compiled directly from the definitions/ + // directory, so referencing one from actions.yaml is the same mistake as + // for any other action type and must produce the same clear error rather + // than silently loading a malformed, duplicated action. + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- dataPreparation: + filename: prep.dp.sqlx` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + const errorMessages = result.compile.compiledGraph.graphErrors.compilationErrors.map( + ({ message }) => message + ); + expect(errorMessages).to.have.lengthOf(1); + expect(errorMessages[0]).to.include(`Action config "dataPreparation" has filename`); + expect(errorMessages[0]).to.include(".sqlx files cannot be referenced from actions.yaml"); + }); + }); + test(`filenames with non-UTF8 characters are valid`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( From cce5b9ff3c86cb98b791c4a1c047f2ec97d0dcf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:04:54 +0200 Subject: [PATCH 31/42] Bump tmp from 0.2.4 to 0.2.7 (#2183) Bumps [tmp](https://github.com/raszi/node-tmp) from 0.2.4 to 0.2.7. - [Changelog](https://github.com/raszi/node-tmp/blob/master/CHANGELOG.md) - [Commits](https://github.com/raszi/node-tmp/compare/v0.2.4...v0.2.7) --- updated-dependencies: - dependency-name: tmp dependency-version: 0.2.6 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d63ebcd57..3ba2e0b41 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "semver": "^7.5.2", "source-map-support": "^0.5.21", "tarjan-graph": "^2.0.0", - "tmp": "^0.2.4", + "tmp": "^0.2.7", "ts-loader": "^5.3.1", "ts-mockito": "^2.6.1", "tslint": "^5.14.0", diff --git a/yarn.lock b/yarn.lock index 41a61ffe5..992deaf55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4144,10 +4144,10 @@ tmp@0.0.29: dependencies: os-tmpdir "~1.0.1" -tmp@^0.2.1, tmp@^0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.4.tgz#c6db987a2ccc97f812f17137b36af2b6521b0d13" - integrity sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ== +tmp@^0.2.1, tmp@^0.2.7: + version "0.2.7" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.7.tgz#26f4db11d1601ce8012dcb8a798ece1c06a99059" + integrity sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw== to-object-path@^0.3.0: version "0.3.0" From 6da16ea400b1b83f9d8b12b360b549a8d621fc8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:21:34 +0200 Subject: [PATCH 32/42] Bump @tootallnate/once from 2.0.0 to 2.0.1 (#2171) Bumps [@tootallnate/once](https://github.com/TooTallNate/once) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/TooTallNate/once/releases) - [Changelog](https://github.com/TooTallNate/once/blob/v2.0.1/CHANGELOG.md) - [Commits](https://github.com/TooTallNate/once/compare/2.0.0...v2.0.1) --- updated-dependencies: - dependency-name: "@tootallnate/once" dependency-version: 2.0.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 992deaf55..b77c4da48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -333,9 +333,9 @@ picomatch "^2.2.2" "@tootallnate/once@2": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" - integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== + version "2.0.1" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.1.tgz#35adc6222e3662fa2222ce123b961476a746b9ea" + integrity sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ== "@types/caseless@*": version "0.12.2" From 7b4c0451d272359855033ad33c30a3e96377ea50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:50:16 +0200 Subject: [PATCH 33/42] Bump fast-uri from 3.0.6 to 3.1.2 (#2161) Bumps [fast-uri](https://github.com/fastify/fast-uri) from 3.0.6 to 3.1.2. - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.0.6...v3.1.2) --- updated-dependencies: - dependency-name: fast-uri dependency-version: 3.1.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index b77c4da48..a3c1ee3e3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1840,9 +1840,9 @@ fast-levenshtein@~2.0.4: integrity "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" fast-uri@^3.0.1: - version "3.0.6" - resolved "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz#88f130b77cfaea2378d56bf970dea21257a68748" - integrity sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw== + version "3.1.2" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.2.tgz#8af3d4fc9d3e71b11572cc2673b514a7d1a8c8ec" + integrity sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ== fast-xml-builder@^1.1.4: version "1.1.4" From aaefb9da114c9f04cab206b94e5ab093a29922d3 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:19:19 -0500 Subject: [PATCH 34/42] remove extra utils_test.ts from api build --- cli/api/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/api/BUILD b/cli/api/BUILD index 39bfea237..d989cfe2c 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -63,7 +63,6 @@ ts_test_suite( "commands/jit/rpc_test.ts", "dbadapters/bigquery_test.ts", "execution_sql_test.ts", - "utils_test.ts", ], data = [ ":node_modules", From 6b3fd34fd8c59f87e2c1735f266e1b8d0b73351c Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:20:06 -0500 Subject: [PATCH 35/42] allow pbts command to run on mac --- tools/ts_proto_library.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ts_proto_library.bzl b/tools/ts_proto_library.bzl index c67a2bb45..5d80120e3 100644 --- a/tools/ts_proto_library.bzl +++ b/tools/ts_proto_library.bzl @@ -31,7 +31,7 @@ def _run_pbts(actions, executable, js_file): # # Run pbts and then patch 'import Long = require("long");' to 'import Long from "long";' # to avoid syntax errors in older rollup-plugin-dts. - command = "{pbts} --out {out} {input} && sed -i 's/import Long = require(\"long\");/import Long from \"long\";/g' {out}".format( + command = "{pbts} --out {out} {input} && sed 's/import Long = require(\"long\");/import Long from \"long\";/g' {out} > {out}.tmp && mv {out}.tmp {out}".format( pbts = executable._pbts.path, out = ts_file.path, input = js_file.path From 5c4d7c54ece6ce4d30879b2ab4dd3e32f1c98e95 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:22:17 -0500 Subject: [PATCH 36/42] refactor concatenateQueries to be a bit more readable --- cli/api/dbadapters/tasks.ts | 77 +++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/cli/api/dbadapters/tasks.ts b/cli/api/dbadapters/tasks.ts index 60344fca6..9baa337e4 100644 --- a/cli/api/dbadapters/tasks.ts +++ b/cli/api/dbadapters/tasks.ts @@ -1,44 +1,53 @@ import { dataform } from "df/protos/ts"; +function formatStatement( + statement: string, + isLast: boolean, + modifier?: (mod: string) => string +) { + // Only treat a comment as trailing if it lives on the statement's last line. + // Searching from the last newline avoids misreading an interior comment (e.g. + // inside a multi-line procedure body) as a trailing comment. + const lastLineStart = statement.lastIndexOf("\n") + 1; + const commentIndex = statement.indexOf("--", lastLineStart); + + let code: string; + let comment: string; + + if (commentIndex === 0) { + code = ""; + comment = statement; + } else if (commentIndex > 0) { + code = statement.substring(0, commentIndex).trimEnd(); + comment = statement.substring(commentIndex); + } else { + code = statement; + comment = ""; + } + + if (modifier && code.length > 0) { + code = modifier(code); + } + + if (!isLast && code.length > 0 && !code.endsWith(";")) { + code = code + ";"; + } + + if (code.length > 0 && comment.length > 0) { + return code + " " + comment; + } + return code || comment; +} + export function concatenateQueries(statements: string[], modifier?: (mod: string) => string) { const processed = statements .filter(statement => !!statement) .map(statement => statement.trim()) .filter(statement => statement.length > 0); - - return processed - .map((statement, index) => { - const isLast = index === processed.length - 1; - const commentIndex = statement.indexOf("--"); - - let code: string; - let comment: string; - - if (commentIndex === 0) { - code = ""; - comment = statement; - } else if (commentIndex > 0) { - code = statement.substring(0, commentIndex).trimEnd(); - comment = statement.substring(commentIndex); - } else { - code = statement; - comment = ""; - } - - if (modifier && code.length > 0) { - code = modifier(code); - } - - if (!isLast && code.length > 0 && !code.endsWith(";")) { - code = code + ";"; - } - - if (code.length > 0 && comment.length > 0) { - return code + " " + comment; - } - return code || comment; - }) - .join("\n"); + let formattedStatements = processed.map((statement, index) => + formatStatement(statement, index === processed.length - 1, modifier) + ); + return formattedStatements.join("\n"); } export class Tasks { From 60bfef362592233fd5e47b28740c8f3fe9caa7e8 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:22:50 -0500 Subject: [PATCH 37/42] update on_schema_change tests to reflect the new format --- cli/api/goldens/on_schema_change_extend.sql | 6 ++---- cli/api/goldens/on_schema_change_fail.sql | 6 ++---- cli/api/goldens/on_schema_change_synchronize.sql | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cli/api/goldens/on_schema_change_extend.sql b/cli/api/goldens/on_schema_change_extend.sql index 21f9558a8..de070956e 100644 --- a/cli/api/goldens/on_schema_change_extend.sql +++ b/cli/api/goldens/on_schema_change_extend.sql @@ -61,8 +61,7 @@ END IF; -- Cleanup temporary tables. DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; -END -; +END; BEGIN CALL `project-id.dataset-id.df_osc_test_uuid`(); EXCEPTION WHEN ERROR THEN @@ -70,8 +69,7 @@ EXCEPTION WHEN ERROR THEN DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; RAISE; END; -DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` -; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; insert into `project-id.dataset-id.incremental_on_schema_change` (`id`,`field1`) select `id`,`field1` diff --git a/cli/api/goldens/on_schema_change_fail.sql b/cli/api/goldens/on_schema_change_fail.sql index ea8d90542..85ad5ed22 100644 --- a/cli/api/goldens/on_schema_change_fail.sql +++ b/cli/api/goldens/on_schema_change_fail.sql @@ -52,8 +52,7 @@ END IF; -- Cleanup temporary tables. DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; -END -; +END; BEGIN CALL `project-id.dataset-id.df_osc_test_uuid`(); EXCEPTION WHEN ERROR THEN @@ -61,8 +60,7 @@ EXCEPTION WHEN ERROR THEN DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; RAISE; END; -DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` -; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; insert into `project-id.dataset-id.incremental_on_schema_change` (`id`,`field1`) select `id`,`field1` diff --git a/cli/api/goldens/on_schema_change_synchronize.sql b/cli/api/goldens/on_schema_change_synchronize.sql index bcfc63eac..ea6bf461b 100644 --- a/cli/api/goldens/on_schema_change_synchronize.sql +++ b/cli/api/goldens/on_schema_change_synchronize.sql @@ -76,8 +76,7 @@ END IF; -- Cleanup temporary tables. DROP TABLE IF EXISTS `project-id.dataset-id.incremental_on_schema_change_df_temp_test_uuid_empty`; -END -; +END; BEGIN CALL `project-id.dataset-id.df_osc_test_uuid`(); EXCEPTION WHEN ERROR THEN @@ -85,8 +84,7 @@ EXCEPTION WHEN ERROR THEN DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; RAISE; END; -DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid` -; +DROP PROCEDURE IF EXISTS `project-id.dataset-id.df_osc_test_uuid`; merge `project-id.dataset-id.incremental_on_schema_change` T using (select 1 as id, 'a' as field1, 'new' as field2 ) S From 585f78644a1e7a987b3f51e7b7170772e75feccc Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:23:29 -0500 Subject: [PATCH 38/42] update format for api spec --- tests/api/api.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/api/api.spec.ts b/tests/api/api.spec.ts index b29936140..3dbdf98fe 100644 --- a/tests/api/api.spec.ts +++ b/tests/api/api.spec.ts @@ -225,7 +225,7 @@ suite("@dataform/api", () => { expect(action.tasks).eql([ dataform.ExecutionTask.create({ type: "statement", - statement: "preOp\n;\ncreate or replace table `schema.a` as foo\n;\npostOp" + statement: "preOp;\ncreate or replace table `schema.a` as foo;\npostOp" }) ]); }); @@ -242,7 +242,7 @@ suite("@dataform/api", () => { dataform.ExecutionTask.create({ type: "statement", statement: - "incremental preOp\n;\ndrop view if exists `schema.a`\n;\ninsert into `schema.a`\t\n()\t\nselect \t\nfrom (incremental foo) as insertions\n;\nincremental postOp" + "incremental preOp;\ndrop view if exists `schema.a`;\ninsert into `schema.a`\t\n()\t\nselect \t\nfrom (incremental foo) as insertions;\nincremental postOp" }) ]); }); From 3c7bbbc7047d6b518546820eeeaf8f15f64732e8 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:24:39 -0500 Subject: [PATCH 39/42] replace drop table in statement comment test because it was causing a test failure --- .../example_trailing_statement_comment_with_operations.sqlx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx b/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx index 4cf7a6b65..dc47e43f5 100644 --- a/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx +++ b/tests/integration/bigquery_project/definitions/example_trailing_statement_comment_with_operations.sqlx @@ -6,7 +6,7 @@ config { } post_operations { - drop table ${self()} + select * from ${self()} } select 1 as col1 -- test comment \ No newline at end of file From 59a5ebde41f4b841518cabb11147a6e7cf5cdb3b Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 12:55:22 -0500 Subject: [PATCH 40/42] add new tests for comments within a query --- cli/api/tasks_test.ts | 6 ++++++ .../definitions/example_internal_comment.sqlx | 12 ++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/integration/bigquery_project/definitions/example_internal_comment.sqlx diff --git a/cli/api/tasks_test.ts b/cli/api/tasks_test.ts index 02485ca96..c790b607d 100644 --- a/cli/api/tasks_test.ts +++ b/cli/api/tasks_test.ts @@ -40,6 +40,12 @@ suite("concatenateQueries", () => { ); }); + test("preserves comment-only lines within a query", () => { + expect(concatenateQueries(["SELECT 1\n--test\nFROM UNNEST([]);"])).deep.equals( + "SELECT 1\n--test\nFROM UNNEST([]);" + ); + }); + test("trims whitespace from statements", () => { expect(concatenateQueries([" SELECT 1 ", " SELECT 2 ", " SELECT 3 -- blah"])).deep.equals( "SELECT 1;\nSELECT 2;\nSELECT 3 -- blah" diff --git a/tests/integration/bigquery_project/definitions/example_internal_comment.sqlx b/tests/integration/bigquery_project/definitions/example_internal_comment.sqlx new file mode 100644 index 000000000..532f05d32 --- /dev/null +++ b/tests/integration/bigquery_project/definitions/example_internal_comment.sqlx @@ -0,0 +1,12 @@ +config { + type: "table", + description: "Verifies fix for https://github.com/dataform-co/dataform/issues/1231" +} + +post_operations { + select * from ${self()} +} + +select 1 as col1 +-- test comment +FROM UNNEST([]) \ No newline at end of file From 4ceac97b0a45e82d361dac6c3ff9c4cf1dc8cfb1 Mon Sep 17 00:00:00 2001 From: Stephen Date: Wed, 8 Jul 2026 12:56:27 -0500 Subject: [PATCH 41/42] update test count in bigquery spec --- tests/integration/bigquery.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/bigquery.spec.ts b/tests/integration/bigquery.spec.ts index 88d4ded4c..af3483a7b 100644 --- a/tests/integration/bigquery.spec.ts +++ b/tests/integration/bigquery.spec.ts @@ -35,7 +35,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) const executedGraph = await dfapi.run(dbadapter, executionGraph).result(); const actionMap = keyBy(executedGraph.actions, v => targetAsReadableString(v.target)); - expect(Object.keys(actionMap).length).eql(17); + expect(Object.keys(actionMap).length).eql(20); // Check the status of action execution. const expectedFailedActions = [ From c19998f1d05b98215014d02a27a8bf654b6f7b34 Mon Sep 17 00:00:00 2001 From: Stephen Patel Date: Wed, 8 Jul 2026 13:10:29 -0500 Subject: [PATCH 42/42] multi line comment tests --- cli/api/dbadapters/tasks.ts | 2 +- cli/api/tasks_test.ts | 8 +++++++- ...lx => example_internal_comment_in_query.sqlx} | 16 ++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) rename tests/integration/bigquery_project/definitions/{example_internal_comment.sqlx => example_internal_comment_in_query.sqlx} (53%) diff --git a/cli/api/dbadapters/tasks.ts b/cli/api/dbadapters/tasks.ts index 9baa337e4..353fcaab4 100644 --- a/cli/api/dbadapters/tasks.ts +++ b/cli/api/dbadapters/tasks.ts @@ -44,7 +44,7 @@ export function concatenateQueries(statements: string[], modifier?: (mod: string .filter(statement => !!statement) .map(statement => statement.trim()) .filter(statement => statement.length > 0); - let formattedStatements = processed.map((statement, index) => + const formattedStatements = processed.map((statement, index) => formatStatement(statement, index === processed.length - 1, modifier) ); return formattedStatements.join("\n"); diff --git a/cli/api/tasks_test.ts b/cli/api/tasks_test.ts index c790b607d..2510e6d6a 100644 --- a/cli/api/tasks_test.ts +++ b/cli/api/tasks_test.ts @@ -40,12 +40,18 @@ suite("concatenateQueries", () => { ); }); - test("preserves comment-only lines within a query", () => { + test("preserves single line comments within a query", () => { expect(concatenateQueries(["SELECT 1\n--test\nFROM UNNEST([]);"])).deep.equals( "SELECT 1\n--test\nFROM UNNEST([]);" ); }); + test("preserves multi line comments within a query", () => { + expect(concatenateQueries(["SELECT 1\n\*multi\nline\ncomment\n*/\nFROM UNNEST([]);"])).deep.equals( + "SELECT 1\n\*multi\nline\ncomment\n*/\nFROM UNNEST([]);" + ); + }); + test("trims whitespace from statements", () => { expect(concatenateQueries([" SELECT 1 ", " SELECT 2 ", " SELECT 3 -- blah"])).deep.equals( "SELECT 1;\nSELECT 2;\nSELECT 3 -- blah" diff --git a/tests/integration/bigquery_project/definitions/example_internal_comment.sqlx b/tests/integration/bigquery_project/definitions/example_internal_comment_in_query.sqlx similarity index 53% rename from tests/integration/bigquery_project/definitions/example_internal_comment.sqlx rename to tests/integration/bigquery_project/definitions/example_internal_comment_in_query.sqlx index 532f05d32..7ff728aef 100644 --- a/tests/integration/bigquery_project/definitions/example_internal_comment.sqlx +++ b/tests/integration/bigquery_project/definitions/example_internal_comment_in_query.sqlx @@ -4,9 +4,21 @@ config { } post_operations { - select * from ${self()} + select * + -- single line comment + /* + multi + line + comment + */ + from ${self()} } select 1 as col1 --- test comment +-- single line comment +/* +multi +line +comment +*/ FROM UNNEST([]) \ No newline at end of file