Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
#!/bin/bash
set -e

echo "Running schema evolution (position) test..."

SCENARIO_DIR="{{SCENARIO_DIR}}"
INPUT_PATH="${SCENARIO_DIR}/../insert-scan/input.parquet"

# Extract ordered column names from "describe -s" output into a comma-joined string.
# Each column line looks like: " 1: sepal.length: optional double"
extract_order() {
grep -E '^[[:space:]]+[0-9]+: [A-Za-z0-9._]+:' "$1" \
| sed -E 's/^[[:space:]]+[0-9]+: ([A-Za-z0-9._]+):.*/\1/' \
| paste -s -d, -
}

assert_order() {
local label="$1"
local expected="$2"
local actual="$3"
if [ "$expected" != "$actual" ]; then
echo "FAIL ${label}: expected '${expected}' got '${actual}'"
exit 1
fi
echo "OK ${label}: ${actual}"
}

{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME}
{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table ${TABLE_NAME} "file://${INPUT_PATH}"
echo "OK Inserted data into ${TABLE_NAME}"

{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/sepos_initial.txt
assert_order "initial order" \
"sepal.length,sepal.width,petal.length,petal.width,variety" \
"$(extract_order /tmp/sepos_initial.txt)"

# --- after ---
{{ICE_CLI}} --config {{CLI_CONFIG}} alter-table ${TABLE_NAME} \
$'[{"op":"add_column","name":"a_after","type":"string","after":"petal.length"}]'
{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/sepos_after.txt
assert_order "after=petal.length" \
"sepal.length,sepal.width,petal.length,a_after,petal.width,variety" \
"$(extract_order /tmp/sepos_after.txt)"

# --- before ---
{{ICE_CLI}} --config {{CLI_CONFIG}} alter-table ${TABLE_NAME} \
$'[{"op":"add_column","name":"a_before","type":"string","before":"variety"}]'
{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/sepos_before.txt
assert_order "before=variety" \
"sepal.length,sepal.width,petal.length,a_after,petal.width,a_before,variety" \
"$(extract_order /tmp/sepos_before.txt)"

# --- first ---
{{ICE_CLI}} --config {{CLI_CONFIG}} alter-table ${TABLE_NAME} \
$'[{"op":"add_column","name":"a_first","type":"string","first":true}]'
{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/sepos_first.txt
assert_order "first=true" \
"a_first,sepal.length,sepal.width,petal.length,a_after,petal.width,a_before,variety" \
"$(extract_order /tmp/sepos_first.txt)"

# --- both after and before: CLI applies after only (before ignored) ---
{{ICE_CLI}} --config {{CLI_CONFIG}} alter-table ${TABLE_NAME} \
$'[{"op":"add_column","name":"bad","type":"string","after":"variety","before":"petal.width"}]'
{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/sepos_conflict.txt
assert_order "after wins when both after and before set" \
"a_first,sepal.length,sepal.width,petal.length,a_after,petal.width,a_before,variety,bad" \
"$(extract_order /tmp/sepos_conflict.txt)"
echo "OK conflicting position: after wins (before ignored)"

# Cleanup
{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME}
{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME}
echo "OK Cleanup done"

echo "Schema evolution (position) test completed successfully"
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
name: "Schema evolution - add_column position (after/before/first)"
description: "Tests alter-table add_column with after/before/first placement; verifies order via describe -s"

catalogConfig:
warehouse: "s3://test-bucket/warehouse"

env:
NAMESPACE_NAME: "test_schema_ev_pos"
TABLE_NAME: "test_schema_ev_pos.iris_pos"
5 changes: 4 additions & 1 deletion ice/src/main/java/com/altinity/ice/cli/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,10 @@ void alterTable(
e.g. [{"op":"drop_column","name":"foo"}]

Supported operations:
- add_column (params: "name", "type" (https://iceberg.apache.org/spec/#primitive-types), "doc" (optional))
- add_column (params: "name", "type" (https://iceberg.apache.org/spec/#primitive-types), "doc" (optional),
"after"/"before"/"first" (optional, at most one;
position the new column after/before a named column,
or at the start of the schema))
- alter_column (params: "name", "type" (https://iceberg.apache.org/spec/#primitive-types))
- rename_column (params: "name", "new_name")
- drop_column (params: "name")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,23 @@ public static class AddColumn extends Update {
private final String name;
private final Type type;
@Nullable private final String doc;
@Nullable private final String after;
@Nullable private final String before;
private final boolean first;

public AddColumn(
@JsonProperty(value = "name", required = true) String name,
@JsonProperty(value = "type", required = true) String type,
@JsonProperty("doc") @Nullable String doc) {
@JsonProperty("doc") @Nullable String doc,
@JsonProperty("after") @Nullable String after,
@JsonProperty("before") @Nullable String before,
@JsonProperty("first") @Nullable Boolean first) {
this.name = name;
this.type = Types.fromPrimitiveString(type);
this.doc = doc;
this.after = after;
this.before = before;
this.first = first != null && first;
}
}

Expand Down Expand Up @@ -138,7 +147,15 @@ public static void run(Catalog catalog, TableIdentifier tableId, List<Update> up
switch (update) {
case AddColumn up -> {
// TODO: support nested columns
schemaUpdates.getValue().addColumn(up.name, up.type, up.doc);
UpdateSchema us = schemaUpdates.getValue();
us.addColumn(up.name, up.type, up.doc);
if (up.after != null) {
us.moveAfter(up.name, up.after);
} else if (up.before != null) {
us.moveBefore(up.name, up.before);
} else if (up.first) {
us.moveFirst(up.name);
}
}
case AlterColumn up -> {
// TODO: support nested columns
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,4 +116,61 @@ public void testDropAllPartitionFields() throws Exception {
table = catalog.loadTable(tableId);
assertThat(table.spec().fields()).isEmpty();
}

@Test
public void testAddColumnAfter() throws Exception {
catalog.buildTable(tableId, schema).create();

List<AlterTable.Update> updates =
Arrays.asList(new AlterTable.AddColumn("age", "long", null, "name", null, null));

AlterTable.run(catalog, tableId, updates);

Table table = catalog.loadTable(tableId);
assertThat(table.schema().columns().stream().map(Types.NestedField::name).toList())
.containsExactly("id", "name", "age", "timestamp_col", "date_col");
}

@Test
public void testAddColumnBefore() throws Exception {
catalog.buildTable(tableId, schema).create();

List<AlterTable.Update> updates =
Arrays.asList(new AlterTable.AddColumn("age", "long", null, null, "timestamp_col", null));

AlterTable.run(catalog, tableId, updates);

Table table = catalog.loadTable(tableId);
assertThat(table.schema().columns().stream().map(Types.NestedField::name).toList())
.containsExactly("id", "name", "age", "timestamp_col", "date_col");
}

@Test
public void testAddColumnFirst() throws Exception {
catalog.buildTable(tableId, schema).create();

List<AlterTable.Update> updates =
Arrays.asList(new AlterTable.AddColumn("age", "long", null, null, null, true));

AlterTable.run(catalog, tableId, updates);

Table table = catalog.loadTable(tableId);
assertThat(table.schema().columns().get(0).name()).isEqualTo("age");
assertThat(table.schema().columns().stream().map(Types.NestedField::name).toList())
.containsExactly("age", "id", "name", "timestamp_col", "date_col");
}

@Test
public void testAddColumnAfterWinsWhenBothAfterAndBeforeSet() throws Exception {
catalog.buildTable(tableId, schema).create();

List<AlterTable.Update> updates =
Arrays.asList(new AlterTable.AddColumn("bad", "string", null, "name", "id", null));

AlterTable.run(catalog, tableId, updates);

Table table = catalog.loadTable(tableId);
assertThat(table.schema().columns().stream().map(Types.NestedField::name).toList())
.containsExactly("id", "name", "bad", "timestamp_col", "date_col");
}
}
Loading