From 11431acc15dc85e0134f04ef9330004dcb8be30c Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Fri, 10 Apr 2026 13:29:14 -0700 Subject: [PATCH 1/2] Fix delta external table schema diff, script syntax, and cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for delta external tables: 1. Schema perpetual diff: The bulk loader populates the schema for all external tables from the cluster, but delta tables auto-infer their schema from the delta log — so YAML configs intentionally omit it. This caused a perpetual diff on every run. Fix: before comparing, clear the cluster-side schema for delta tables when the YAML doesn't specify one. If the YAML provides a custom schema, keep it for proper comparison. 2. Missing closing paren: CreateDeltaScript generated an unclosed parenthesis around the connection string, producing invalid KQL that fails Kusto syntax validation and cannot be applied. 3. Remove unnecessary dataFormat requirement: Delta tables don't use dataformat= in the Kusto command syntax. Removed the misleading validation that required it. 4. Only emit with() properties when set: Delta script no longer emits empty folder='', docString='', fileExtension='' when not specified in YAML. The with() block is omitted entirely if no properties are set. Fixes github/data#11973 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- KustoSchemaTools/Changes/DatabaseChanges.cs | 15 +++++++++++++++ KustoSchemaTools/Model/ExternalTable.cs | 13 +++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/KustoSchemaTools/Changes/DatabaseChanges.cs b/KustoSchemaTools/Changes/DatabaseChanges.cs index 0770927..cab0be3 100644 --- a/KustoSchemaTools/Changes/DatabaseChanges.cs +++ b/KustoSchemaTools/Changes/DatabaseChanges.cs @@ -113,6 +113,21 @@ public static List GenerateChanges(Database oldState, Database newState result.AddRange(mvChanges); result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ContinuousExports, nameof(newState.ContinuousExports), log)); result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Functions, nameof(newState.Functions), log)); + + // For delta external tables, the schema is auto-inferred from the delta log. + // If the YAML doesn't specify a schema, clear the cluster-side schema so it + // doesn't cause a perpetual diff. If the YAML does specify a schema, keep the + // cluster-side schema for proper comparison. + foreach (var et in newState.ExternalTables) + { + if (et.Value.Kind?.ToLower() == "delta" + && et.Value.Schema?.Any() != true + && oldState.ExternalTables.ContainsKey(et.Key)) + { + oldState.ExternalTables[et.Key].Schema = null; + } + } + result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ExternalTables, nameof(newState.ExternalTables), log)); if (newState.EntityGroups.Any()) diff --git a/KustoSchemaTools/Model/ExternalTable.cs b/KustoSchemaTools/Model/ExternalTable.cs index 11bc7d3..5c890f2 100644 --- a/KustoSchemaTools/Model/ExternalTable.cs +++ b/KustoSchemaTools/Model/ExternalTable.cs @@ -137,8 +137,6 @@ private string CreateSqlScript(string name) private string CreateDeltaScript(string name) { - if (string.IsNullOrWhiteSpace(DataFormat)) throw new ArgumentException("DataFormat can't be empty"); - var sb = new StringBuilder(); sb.AppendLine($".create-or-alter external table {name}"); if (Schema?.Any() == true) @@ -146,11 +144,18 @@ private string CreateDeltaScript(string name) sb.AppendLine($"({string.Join(", ", Schema.Select(c => $"{c.Key.BracketIfIdentifier()}:{c.Value}"))})"); } sb.AppendLine("kind=delta"); - sb.AppendLine($"(h@'{ConnectionString}'"); + sb.AppendLine($"(h@'{ConnectionString}')"); + var withProps = new List(); + if (!string.IsNullOrEmpty(Folder)) withProps.Add($"folder='{Folder}'"); + if (!string.IsNullOrEmpty(DocString)) withProps.Add($"docString='{DocString}'"); var ext = string.IsNullOrWhiteSpace(FileExtensions) ? "" : FileExtensions.StartsWith(".") ? FileExtensions : "." + FileExtensions; + if (!string.IsNullOrEmpty(ext)) withProps.Add($"fileExtension='{ext}'"); - sb.AppendLine($"with(folder='{Folder}', docString='{DocString}', fileExtension='{ext}') "); + if (withProps.Any()) + { + sb.AppendLine($"with({string.Join(", ", withProps)})"); + } return sb.ToString(); } From c2e88ace06d524db05d7ded6dac06f0751f45398 Mon Sep 17 00:00:00 2001 From: Ashley Van Spankeren <25673124+ashleyvansp@users.noreply.github.com> Date: Mon, 13 Apr 2026 09:12:50 -0700 Subject: [PATCH 2/2] Address comments --- KustoSchemaTools/Changes/DatabaseChanges.cs | 5 +++-- KustoSchemaTools/Model/ExternalTable.cs | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/KustoSchemaTools/Changes/DatabaseChanges.cs b/KustoSchemaTools/Changes/DatabaseChanges.cs index cab0be3..068066e 100644 --- a/KustoSchemaTools/Changes/DatabaseChanges.cs +++ b/KustoSchemaTools/Changes/DatabaseChanges.cs @@ -122,9 +122,10 @@ public static List GenerateChanges(Database oldState, Database newState { if (et.Value.Kind?.ToLower() == "delta" && et.Value.Schema?.Any() != true - && oldState.ExternalTables.ContainsKey(et.Key)) + && oldState.ExternalTables.TryGetValue(et.Key, out var oldExternalTable) + && oldExternalTable.Kind?.ToLower() == "delta") { - oldState.ExternalTables[et.Key].Schema = null; + oldExternalTable.Schema = null; } } diff --git a/KustoSchemaTools/Model/ExternalTable.cs b/KustoSchemaTools/Model/ExternalTable.cs index 5c890f2..d85768a 100644 --- a/KustoSchemaTools/Model/ExternalTable.cs +++ b/KustoSchemaTools/Model/ExternalTable.cs @@ -137,6 +137,8 @@ private string CreateSqlScript(string name) private string CreateDeltaScript(string name) { + if (string.IsNullOrWhiteSpace(ConnectionString)) throw new ArgumentException("ConnectionString can't be empty"); + var sb = new StringBuilder(); sb.AppendLine($".create-or-alter external table {name}"); if (Schema?.Any() == true)