diff --git a/cli/index.ts b/cli/index.ts index 452d71f5f..ae6b4b7b5 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -5,7 +5,7 @@ import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; -import { build, compile, credentials, init, install, run, test } from "df/cli/api"; +import { build, compile, credentials, init, install, prune, run, test } from "df/cli/api"; import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { prettyJsonStringify } from "df/cli/api/utils"; @@ -393,6 +393,10 @@ export function runCli() { dotOutputOption, timeoutOption, quietCompileOption, + actionsOption, + tagsOption, + includeDepsOption, + includeDependentsOption, { name: verboseOptionName, option: { @@ -429,7 +433,24 @@ export function runCli() { timeoutMillis: argv[timeoutOption.name] || undefined, verbose: argv[verboseOptionName] || false }); - printCompiledGraph(compiledGraph, outputType, argv[quietCompileOption.name]); + + // The whole project must compile (ref() resolution needs every action + // registered), but the printed output can be filtered to the selected + // action(s) -- mirroring how `run`/`build` prune the graph. We only prune + // a clean graph; if compilation produced errors we print the full graph + // plus the errors, keeping graph-level errors as-is. + const hasSelector = + argv[actionsOption.name]?.length > 0 || argv[tagsOption.name]?.length > 0; + const outputGraph = + hasSelector && !compiledGraphHasErrors(compiledGraph) + ? prune(compiledGraph, { + actions: argv[actionsOption.name], + tags: argv[tagsOption.name], + includeDependencies: argv[includeDepsOption.name], + includeDependents: argv[includeDependentsOption.name] + }) + : compiledGraph; + printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]); if (compiledGraphHasErrors(compiledGraph)) { print(""); printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); diff --git a/cli/index_compile_test.ts b/cli/index_compile_test.ts index 4f82799b7..c59c7576d 100644 --- a/cli/index_compile_test.ts +++ b/cli/index_compile_test.ts @@ -361,6 +361,139 @@ SELECT 1 as id }); }); +suite("compile node selection", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + // Builds a project with three tables: upstream -> midstream -> downstream. + async function setupSelectionProject(): Promise { + const projectDir = tmpDirFixture.createNewTmpDir(); + const npmCacheDir = tmpDirFixture.createNewTmpDir(); + + await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + ); + + const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); + const workflowSettings = dataform.WorkflowSettings.create( + loadYaml(fs.readFileSync(workflowSettingsPath, "utf8")) + ); + delete workflowSettings.dataformCoreVersion; + fs.writeFileSync(workflowSettingsPath, dumpYaml(workflowSettings)); + + fs.writeFileSync( + path.join(projectDir, "package.json"), + `{ + "dependencies":{ + "@dataform/core": "${version}" + } +}` + ); + await getProcessResult( + execFile(npmPath, [ + "install", + "--prefix", + projectDir, + "--cache", + npmCacheDir, + corePackageTarPath + ]) + ); + + const def = (name: string, contents: string) => { + const filePath = path.join(projectDir, "definitions", `${name}.sqlx`); + fs.ensureFileSync(filePath); + fs.writeFileSync(filePath, contents); + }; + def("upstream", `config { type: "table", tags: ["daily"] }\nSELECT 1 AS id`); + def("midstream", `config { type: "table" }\nSELECT * FROM \${ref("upstream")}`); + def("downstream", `config { type: "table" }\nSELECT * FROM \${ref("midstream")}`); + + return projectDir; + } + + const tableNames = (stdout: string): string[] => + JSON.parse(stdout).tables.map((table: any) => table.target.name).sort(); + + test("no selector emits the entire graph", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream", "upstream"]); + }); + + test("--actions filters output to the selected action", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--actions", "midstream", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["midstream"]); + }); + + test("--actions --include-deps pulls in upstream dependencies", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "compile", + projectDir, + "--actions", + "midstream", + "--include-deps", + "--json" + ]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["midstream", "upstream"]); + }); + + test("--actions --include-dependents pulls in downstream dependents", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [ + cliEntryPointPath, + "compile", + projectDir, + "--actions", + "midstream", + "--include-dependents", + "--json" + ]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["downstream", "midstream"]); + }); + + test("--tags filters output to actions carrying the tag", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--tags", "daily", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals(["upstream"]); + }); + + test("selector matching nothing emits an empty graph and exits zero", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--actions", "nope", "--json"]) + ); + expect(result.exitCode, result.stderr).equals(0); + expect(tableNames(result.stdout)).deep.equals([]); + }); + + test("--include-deps without a selector is rejected", async () => { + const projectDir = await setupSelectionProject(); + const result = await getProcessResult( + execFile(nodePath, [cliEntryPointPath, "compile", projectDir, "--include-deps", "--json"]) + ); + expect(result.exitCode).not.equals(0); + expect(result.stderr).contains("--include-deps"); + }); +}); + suite("extension config", ({ afterEach }) => { const tmpDirFixture = new TmpDirFixture(afterEach);