diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1c735ffb..53250d29 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: - name: Check for changed files id: check - uses: stackabletech/actions/detect-changes@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + uses: stackabletech/actions/detect-changes@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: patterns: | - '.github/workflows/build.yaml' @@ -166,16 +166,16 @@ jobs: - name: Build Container Image id: build - uses: stackabletech/actions/build-container-image@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + uses: stackabletech/actions/build-container-image@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: image-name: ${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }} container-file: docker/Dockerfile - - name: Publish Container Image + - name: Publish Container Image (oci.stackable.tech) if: ${{ !github.event.pull_request.head.repo.fork }} - uses: stackabletech/actions/publish-image@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + uses: stackabletech/actions/publish-image@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -184,6 +184,17 @@ jobs: image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + - name: Publish Container Image (quay.io) + if: ${{ !github.event.pull_request.head.repo.fork }} + uses: stackabletech/actions/publish-image@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased + with: + image-registry-uri: quay.io + image-registry-username: stackable+robot_sdp_test_github_action_build + image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_TEST_GITHUB_ACTION_BUILD_SECRET }} + image-repository: stackable/sdp-test/${{ env.OPERATOR_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + publish-index-manifest: name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index if: | @@ -202,8 +213,8 @@ jobs: with: persist-credentials: false - - name: Publish and Sign Image Index - uses: stackabletech/actions/publish-image-index-manifest@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + - name: Publish and Sign Image Index (oci.stackable.tech) + uses: stackabletech/actions/publish-image-index-manifest@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -211,6 +222,15 @@ jobs: image-repository: sdp/${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} + - name: Publish and Sign Image Index (quay.io) + uses: stackabletech/actions/publish-image-index-manifest@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased + with: + image-registry-uri: quay.io + image-registry-username: stackable+robot_sdp_test_github_action_build + image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_TEST_GITHUB_ACTION_BUILD_SECRET }} + image-repository: stackable/sdp-test/${{ env.OPERATOR_NAME }} + image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} + publish-helm-chart: name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart if: | @@ -229,8 +249,8 @@ jobs: persist-credentials: false submodules: recursive - - name: Package, Publish, and Sign Helm Chart - uses: stackabletech/actions/publish-helm-chart@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + - name: Package, Publish, and Sign Helm Chart (oci.stackable.tech) + uses: stackabletech/actions/publish-helm-chart@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: chart-registry-uri: oci.stackable.tech chart-registry-username: robot$sdp-charts+github-action-build @@ -241,6 +261,18 @@ jobs: app-version: ${{ needs.build-container-image.outputs.operator-version }} publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }} + - name: Package, Publish, and Sign Helm Chart (quay.io) + uses: stackabletech/actions/publish-helm-chart@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased + with: + chart-registry-uri: quay.io + chart-registry-username: stackable+robot_sdp_charts_test_github_action_build + chart-registry-password: ${{ secrets.QUAY_ROBOT_SDP_CHARTS_TEST_GITHUB_ACTION_BUILD_SECRET }} + chart-repository: stackable/sdp-charts-test + chart-directory: deploy/helm/${{ env.OPERATOR_NAME }} + chart-version: ${{ needs.build-container-image.outputs.operator-version }} + app-version: ${{ needs.build-container-image.outputs.operator-version }} + publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }} + openshift-preflight-check: name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} if: | @@ -259,12 +291,18 @@ jobs: - arm64 runs-on: ubuntu-latest steps: - - name: Run OpenShift Preflight Check - uses: stackabletech/actions/run-openshift-preflight@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + - name: Run OpenShift Preflight Check (oci.stackable.tech) + uses: stackabletech/actions/run-openshift-preflight@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} + - name: Run OpenShift Preflight Check (quay.io) + uses: stackabletech/actions/run-openshift-preflight@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased + with: + image-index-uri: quay.io/stackable/sdp-test/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} + image-architecture: ${{ matrix.arch }} + # This job is a required check in GitHub Settings for this repository. # It saves us having to list many required jobs, or work around dynamically # named jobs (since there is no concept of required settings). @@ -300,7 +338,7 @@ jobs: persist-credentials: false - name: Send Notification - uses: stackabletech/actions/send-slack-notification@9848c5593dff4793aacba240116a648c02f20fa4 # v0.13.1 + uses: stackabletech/actions/send-slack-notification@b4f7b1301d2e46e623cd04a22994a327470d4743 # unreleased with: publish-helm-chart-result: ${{ needs.publish-helm-chart.result }} publish-manifests-result: ${{ needs.publish-index-manifest.result }} diff --git a/Cargo.lock b/Cargo.lock index e8d63bf9..580d6a8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,7 +1524,6 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "darling", "regex", @@ -2904,7 +2903,6 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "const-oid", "ecdsa", @@ -2928,7 +2926,6 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.110.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "base64", "clap", @@ -2969,7 +2966,6 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "darling", "proc-macro2", @@ -2980,7 +2976,6 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "jiff", "k8s-openapi", @@ -2997,7 +2992,6 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "axum", "clap", @@ -3021,8 +3015,8 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ + "kube", "schemars", "serde", "serde_json", @@ -3034,7 +3028,6 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "convert_case", "convert_case_extras", @@ -3052,7 +3045,6 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index ceb2dfb4..389ab3ea 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4859,12 +4859,7 @@ rec { crateName = "k8s-version"; version = "0.1.3"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/k8s-version; }; libName = "k8s_version"; authors = [ "Stackable GmbH " @@ -9589,12 +9584,7 @@ rec { crateName = "stackable-certs"; version = "0.4.0"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-certs; }; libName = "stackable_certs"; authors = [ "Stackable GmbH " @@ -9692,12 +9682,7 @@ rec { crateName = "stackable-operator"; version = "0.110.1"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-operator; }; libName = "stackable_operator"; authors = [ "Stackable GmbH " @@ -9872,12 +9857,7 @@ rec { crateName = "stackable-operator-derive"; version = "0.3.1"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-operator-derive; }; procMacro = true; libName = "stackable_operator_derive"; authors = [ @@ -9907,12 +9887,7 @@ rec { crateName = "stackable-shared"; version = "0.1.0"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-shared; }; libName = "stackable_shared"; authors = [ "Stackable GmbH " @@ -9988,12 +9963,7 @@ rec { crateName = "stackable-telemetry"; version = "0.6.3"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-telemetry; }; libName = "stackable_telemetry"; authors = [ "Stackable GmbH " @@ -10098,17 +10068,18 @@ rec { crateName = "stackable-versioned"; version = "0.9.0"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-versioned; }; libName = "stackable_versioned"; authors = [ "Stackable GmbH " ]; dependencies = [ + { + name = "kube"; + packageId = "kube"; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; + } { name = "schemars"; packageId = "schemars"; @@ -10142,12 +10113,7 @@ rec { crateName = "stackable-versioned-macros"; version = "0.9.0"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-versioned-macros; }; procMacro = true; libName = "stackable_versioned_macros"; authors = [ @@ -10210,12 +10176,7 @@ rec { crateName = "stackable-webhook"; version = "0.9.1"; edition = "2024"; - workspace_member = null; - src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; - sha256 = "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z"; - }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ../operator-rs/crates/stackable-webhook; }; libName = "stackable_webhook"; authors = [ "Stackable GmbH " diff --git a/Cargo.toml b/Cargo.toml index 07505e63..6a56f59c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,4 +36,4 @@ tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] # TODO revert this before merging! # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/Makefile b/Makefile index 32f2bcad..6a0a9ff3 100644 --- a/Makefile +++ b/Makefile @@ -76,3 +76,9 @@ run-dev: check-nix check-kubernetes stop-dev: check-nix check-kubernetes nix run --extra-experimental-features "nix-command flakes" -f. tilt -- down + +helm-install: + helm install \ + --values deploy/helm/airflow-operator/values.yaml \ + --values deploy/helm/airflow-operator/values/$(OCI_REGISTRY_HOSTNAME).yaml \ + airflow-operator deploy/helm/airflow-operator diff --git a/Tiltfile b/Tiltfile index 53c3a8e4..fc177e85 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,16 +1,18 @@ -# If tilt_options.json exists read it and load the default_registry value from it +# Load the metadata first, so that we immediately get access to the operator name +meta = read_json('nix/meta.json') +operator_name = meta['operator']['name'] + +# If tilt_options.json exists read it and load the default_registry and default_repository value from it settings = read_json('tilt_options.json', default={}) -registry = settings.get('default_registry', 'oci.stackable.tech/sandbox') +registry = settings.get('default_registry', 'oci.stackable.tech') +repository = settings.get('default_repository', 'sdp' + '/' + operator_name) -# Configure default registry either read from config file above, or with default value of "oci.stackable.tech/sandbox" +# Configure default registry either read from config file above, or with default value of "oci.stackable.tech" default_registry(registry) -meta = read_json('nix/meta.json') -operator_name = meta['operator']['name'] - custom_build( - registry + '/' + operator_name, - 'make regenerate-nix && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + operator_name + '" && ./result/load-image | docker load', + registry + '/' + repository, + 'make regenerate-nix && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + repository + '" && ./result/load-image | docker load', deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], ignore=['*.~undo-tree~'], # ignore=['result*', 'Cargo.nix', 'target', *.yaml], @@ -18,8 +20,8 @@ custom_build( ) # We need to set the correct image annotation on the operator Deployment to use e.g. -# oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of -# oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) +# oci.stackable.tech/sdp/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of +# oci.stackable.tech/sdp/opa-operator:0.0.0-dev (which does not exist) k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') @@ -28,13 +30,15 @@ k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.inte # supported by helm(set). helm_values = settings.get('helm_values', None) -helm_override_image_repository = 'image.repository=' + registry + '/' + operator_name +helm_override_image_registry = 'image.registry=' + registry +helm_override_image_repository = 'image.repository=' + repository k8s_yaml(helm( 'deploy/helm/' + operator_name, name=operator_name, namespace="stackable-operators", set=[ + helm_override_image_registry, helm_override_image_repository, ], values=helm_values, diff --git a/crate-hashes.json b/crate-hashes.json index 7a259ca2..ceeb77ce 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,3 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#k8s-version@0.1.3": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-certs@0.4.0": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator-derive@0.3.1": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator@0.110.1": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-shared@0.1.0": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-telemetry@0.6.3": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned-macros@0.9.0": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned@0.9.0": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-webhook@0.9.1": "14n9a3j3l50iixybfzg4x8gmfxv4idkyvfxmb48d552isjqcg83z", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/.helmignore b/deploy/helm/airflow-operator/.helmignore index fef44b7e..0f7e0c9a 100644 --- a/deploy/helm/airflow-operator/.helmignore +++ b/deploy/helm/airflow-operator/.helmignore @@ -6,6 +6,8 @@ # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. +# Ignore the file itself +.helmignore .DS_Store # Common VCS dirs .git/ @@ -26,3 +28,5 @@ .idea/ *.tmproj .vscode/ +# Partial, unmerged, registry specific values files +values/ diff --git a/deploy/helm/airflow-operator/templates/_helpers.tpl b/deploy/helm/airflow-operator/templates/_helpers.tpl index 1096ffc6..80f2848b 100644 --- a/deploy/helm/airflow-operator/templates/_helpers.tpl +++ b/deploy/helm/airflow-operator/templates/_helpers.tpl @@ -77,3 +77,10 @@ Labels for Kubernetes objects created by helm test {{- define "operator.testLabels" -}} helm.sh/test: {{ include "operator.chart" . }} {{- end }} + +{{/* +Build the full container image reference. +*/}} +{{- define "operator.image" -}} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) -}} +{{- end }} diff --git a/deploy/helm/airflow-operator/templates/deployment.yaml b/deploy/helm/airflow-operator/templates/deployment.yaml index 0f4d902e..7f45a437 100644 --- a/deploy/helm/airflow-operator/templates/deployment.yaml +++ b/deploy/helm/airflow-operator/templates/deployment.yaml @@ -15,7 +15,7 @@ spec: template: metadata: annotations: - internal.stackable.tech/image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + internal.stackable.tech/image: {{ include "operator.image" . }} checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} @@ -37,7 +37,7 @@ spec: - name: {{ include "operator.appname" . }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: {{ include "operator.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- toYaml .Values.resources | nindent 12 }} @@ -70,6 +70,14 @@ spec: - name: OPERATOR_SERVICE_NAME value: {{ include "operator.fullname" . }} + # The image registry, like "oci.stackable.tech". Used to derive product image name. + - name: IMAGE_REGISTRY + value: {{ .Values.image.registry }} + + # The image repository, like "sdp/airflow-operator" + - name: IMAGE_REPOSITORY + value: {{ .Values.image.repository }} + # Operators need to know the node name they are running on, to e.g. discover the # Kubernetes domain name from the kubelet API. - name: KUBERNETES_NODE_NAME diff --git a/deploy/helm/airflow-operator/values.yaml b/deploy/helm/airflow-operator/values.yaml index 89b8a10d..194b3a73 100644 --- a/deploy/helm/airflow-operator/values.yaml +++ b/deploy/helm/airflow-operator/values.yaml @@ -1,7 +1,6 @@ # Default values for airflow-operator. --- image: - repository: oci.stackable.tech/sdp/airflow-operator pullPolicy: IfNotPresent pullSecrets: [] diff --git a/deploy/helm/airflow-operator/values/oci.stackable.tech.yaml b/deploy/helm/airflow-operator/values/oci.stackable.tech.yaml new file mode 100644 index 00000000..06f5dd12 --- /dev/null +++ b/deploy/helm/airflow-operator/values/oci.stackable.tech.yaml @@ -0,0 +1,5 @@ +--- +# Values overlay for chart packages published to oci.stackable.tech. +image: + registry: oci.stackable.tech + repository: sdp/airflow-operator diff --git a/deploy/helm/airflow-operator/values/quay.io.yaml b/deploy/helm/airflow-operator/values/quay.io.yaml new file mode 100644 index 00000000..2288c22f --- /dev/null +++ b/deploy/helm/airflow-operator/values/quay.io.yaml @@ -0,0 +1,5 @@ +--- +# Values overlay for chart packages published to quay.io. +image: + registry: quay.io + repository: stackable/sdp-test/airflow-operator diff --git a/extra/crds.yaml b/extra/crds.yaml index f18b9d79..f4c4ab90 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2398,8 +2398,9 @@ spec: properties: custom: description: |- - Overwrite the docker image. - Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + Overwrite the container image. + + Specify the full container image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -2425,15 +2426,29 @@ spec: type: object nullable: true type: array - repo: - description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + registry: + description: |- + The container image registry, e.g. `oci.stackable.tech`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + repository: + description: |- + The repository on the container image registry where the container image is located, e.g. + `sdp/airflow`. + + If not specified, the operator will use the image registry provided via the operator + environment options. nullable: true type: string stackableVersion: description: |- Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - If not specified, the operator will use its own version, e.g. `23.4.1`. - When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. nullable: true type: string type: object @@ -8379,8 +8394,9 @@ spec: properties: custom: description: |- - Overwrite the docker image. - Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + Overwrite the container image. + + Specify the full container image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -8406,15 +8422,29 @@ spec: type: object nullable: true type: array - repo: - description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + registry: + description: |- + The container image registry, e.g. `oci.stackable.tech`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + repository: + description: |- + The repository on the container image registry where the container image is located, e.g. + `sdp/airflow`. + + If not specified, the operator will use the image registry provided via the operator + environment options. nullable: true type: string stackableVersion: description: |- Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - If not specified, the operator will use its own version, e.g. `23.4.1`. - When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. nullable: true type: string type: object diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index a54c7e4f..450672ae 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -29,6 +29,7 @@ use stackable_operator::{ }, }, }, + cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ product_image_selection::{self, ResolvedProductImage}, @@ -112,13 +113,13 @@ use crate::{ }; pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; -pub const DOCKER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, + pub operator_environment: OperatorEnvironmentOptions, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -380,7 +381,14 @@ pub async fn reconcile_airflow( let resolved_product_image = airflow .spec .image - .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION) + .resolve( + &ctx.operator_environment.image_registry.to_string(), + ctx.operator_environment + .image_repository + // I'm not happy about this. Is this something we can and should adjust? + .trim_end_matches("-operator"), + crate::built_info::PKG_VERSION, + ) .context(ResolveProductImageSnafu)?; let cluster_operation_cond_builder = diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e02c2b9a..901a51fd 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1088,9 +1088,12 @@ pub fn build_recommended_labels<'a, T>( #[cfg(test)] mod tests { - use stackable_operator::commons::product_image_selection::ResolvedProductImage; + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + versioned::test_utils::RoundtripTestData, + }; - use crate::v1alpha2::AirflowCluster; + use crate::{v1alpha1, v1alpha2}; #[test] fn test_cluster_config() { @@ -1119,13 +1122,13 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let cluster: AirflowCluster = + let cluster: v1alpha2::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let resolved_airflow_image: ResolvedProductImage = cluster .spec .image - .resolve("airflow", "0.0.0-dev") + .resolve("oci.example.org", "airflow", "0.0.0-dev") .expect("test: resolved product image is always valid"); assert_eq!("3.1.6", &resolved_airflow_image.product_version); @@ -1136,4 +1139,16 @@ mod tests { // defaults to true assert!(cluster.spec.cluster_config.database_initialization.enabled); } + + impl RoundtripTestData for v1alpha1::AirflowClusterSpec { + fn roundtrip_test_data() -> Vec { + vec![] + } + } + + impl RoundtripTestData for v1alpha2::AirflowClusterSpec { + fn roundtrip_test_data() -> Vec { + vec![] + } + } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 4abf38c3..6cc89e99 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -121,6 +121,17 @@ async fn main() -> anyhow::Result<()> { }, )); + let webhook_server = create_webhook_server( + &operator_environment, + maintenance.disable_crd_maintenance, + client.as_kube_client(), + ) + .await?; + + let webhook_server = webhook_server + .run(sigterm_watcher.handle()) + .map_err(|err| anyhow!(err).context("failed to run webhook server")); + let airflow_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), @@ -170,6 +181,7 @@ async fn main() -> anyhow::Result<()> { airflow_controller::error_policy, Arc::new(airflow_controller::Ctx { client: client.clone(), + operator_environment, product_config, }), ) @@ -192,17 +204,6 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); - let webhook_server = create_webhook_server( - &operator_environment, - maintenance.disable_crd_maintenance, - client.as_kube_client(), - ) - .await?; - - let webhook_server = webhook_server - .run(sigterm_watcher.handle()) - .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let delayed_airflow_controller = async { signal::crd_established(&client, v1alpha1::AirflowCluster::crd_name(), None) .await?;