From c75deb8355f5174c34163160dc8ab56caa3ce703 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 5 Nov 2025 14:30:09 +0100 Subject: [PATCH 1/7] hive 4.0.1 is building --- hive/Dockerfile | 64 ++++++++++++++++++++++++++++++++++++++++++- hive/boil-config.toml | 8 ++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index 46ab0412f..d274ac173 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -3,6 +3,66 @@ FROM local-image/hadoop/hadoop AS hadoop-builder +FROM local-image/java-devel AS hive-metastore-opa-authorizer-builder + +ARG PRODUCT_VERSION +ARG HADOOP_HADOOP_VERSION +ARG HIVE_METASTORE_OPA_AUTHORIZER_VERSION +ARG STACKABLE_USER_UID +# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) +# This can be used to speed up builds when disk space is of no concern. +ARG DELETE_CACHES="true" + +# Reassign the arg to `HADOOP_VERSION` for better readability. +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} + +# We need Java 11 to build the hive-metastore-opa-authorizer +RUN < Date: Wed, 5 Nov 2025 14:37:29 +0100 Subject: [PATCH 2/7] move env var definition --- hive/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index d274ac173..b9121c711 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -250,8 +250,6 @@ COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin/start-metastore /stackab COPY hive/licenses /licenses -ENV HIVE_HOME=/stackable/hive-metastore - RUN < Date: Wed, 5 Nov 2025 14:38:55 +0100 Subject: [PATCH 3/7] added todo --- hive/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hive/Dockerfile b/hive/Dockerfile index b9121c711..dc12cd5de 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -17,6 +17,7 @@ ARG DELETE_CACHES="true" ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} # We need Java 11 to build the hive-metastore-opa-authorizer +# TODO: How can we improve this or e.g. specify different java-dev images? RUN < Date: Thu, 6 Nov 2025 18:05:25 +0100 Subject: [PATCH 4/7] 3.1.3 building and working with patchable --- hive/Dockerfile | 65 +------------------ hive/boil-config.toml | 16 ++--- hive/hive-metastore-opa-authorizer/Dockerfile | 55 ++++++++++++++++ .../boil-config.toml | 8 +++ .../stackable/patches/patchable.toml | 2 + .../stackable/patches/v1.0.0/patchable.toml | 2 + 6 files changed, 78 insertions(+), 70 deletions(-) create mode 100644 hive/hive-metastore-opa-authorizer/Dockerfile create mode 100644 hive/hive-metastore-opa-authorizer/boil-config.toml create mode 100644 hive/hive-metastore-opa-authorizer/stackable/patches/patchable.toml create mode 100644 hive/hive-metastore-opa-authorizer/stackable/patches/v1.0.0/patchable.toml diff --git a/hive/Dockerfile b/hive/Dockerfile index dc12cd5de..fb33fd8f0 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -2,67 +2,7 @@ # check=error=true FROM local-image/hadoop/hadoop AS hadoop-builder - -FROM local-image/java-devel AS hive-metastore-opa-authorizer-builder - -ARG PRODUCT_VERSION -ARG HADOOP_HADOOP_VERSION -ARG HIVE_METASTORE_OPA_AUTHORIZER_VERSION -ARG STACKABLE_USER_UID -# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) -# This can be used to speed up builds when disk space is of no concern. -ARG DELETE_CACHES="true" - -# Reassign the arg to `HADOOP_VERSION` for better readability. -ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} - -# We need Java 11 to build the hive-metastore-opa-authorizer -# TODO: How can we improve this or e.g. specify different java-dev images? -RUN < Date: Thu, 6 Nov 2025 19:49:06 +0100 Subject: [PATCH 5/7] use patchable for all versions --- hive/boil-config.toml | 21 +++------------- hive/hive-metastore-opa-authorizer/Dockerfile | 8 +++---- .../boil-config.toml | 24 ++++++++++++++++--- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/hive/boil-config.toml b/hive/boil-config.toml index 5be76f892..f4d4ed316 100644 --- a/hive/boil-config.toml +++ b/hive/boil-config.toml @@ -4,7 +4,7 @@ java-base = "11" java-devel = "8" "hadoop/hadoop" = "3.3.6" # hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer -"hive/hive-metastore-opa-authorizer" = "v1.0.0" +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-3.1.3-hadoop-3.3.6" [versions."3.1.3".build-arguments] jmx-exporter-version = "1.3.0" @@ -13,28 +13,13 @@ aws-java-sdk-bundle-version = "1.12.367" azure-storage-version = "7.0.1" azure-keyvault-core-version = "1.0.0" -[versions."4.0.0".local-images] -# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 -java-base = "11" -java-devel = "8" -"hadoop/hadoop" = "3.3.6" -# hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer -"hive/hive-metastore-opa-authorizer" = "v1.0.0" - -[versions."4.0.0".build-arguments] -jmx-exporter-version = "1.3.0" -# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 -aws-java-sdk-bundle-version = "1.12.367" -azure-storage-version = "7.0.1" -azure-keyvault-core-version = "1.0.0" - [versions."4.0.1".local-images] # Hive 4.0 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 java-base = "11" java-devel = "8" "hadoop/hadoop" = "3.3.6" # hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer -"hive/hive-metastore-opa-authorizer" = "v1.0.0" +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-4.0.1-hadoop-3.3.6" [versions."4.0.1".build-arguments] jmx-exporter-version = "1.3.0" @@ -49,7 +34,7 @@ java-base = "17" java-devel = "17" "hadoop/hadoop" = "3.4.2" # hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer -"hive/hive-metastore-opa-authorizer" = "v1.0.0" +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-4.1.0-hadoop-3.4.2" [versions."4.1.0".build-arguments] jmx-exporter-version = "1.3.0" diff --git a/hive/hive-metastore-opa-authorizer/Dockerfile b/hive/hive-metastore-opa-authorizer/Dockerfile index 5802075dd..03f3619a0 100644 --- a/hive/hive-metastore-opa-authorizer/Dockerfile +++ b/hive/hive-metastore-opa-authorizer/Dockerfile @@ -3,7 +3,7 @@ FROM local-image/java-devel -ARG PRODUCT_VERSION +ARG AUTHORIZER_VERSION ARG HIVE_VERSION ARG HADOOP_VERSION ARG STACKABLE_USER_UID @@ -15,7 +15,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hive/hive-metastore-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hive/hive-metastore-opa-authorizer/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hive/hive-metastore-opa-authorizer/stackable/patches/${PRODUCT_VERSION} /stackable/src/hive/hive-metastore-opa-authorizer/stackable/patches/${PRODUCT_VERSION} +COPY --chown=${STACKABLE_USER_UID}:0 hive/hive-metastore-opa-authorizer/stackable/patches/${AUTHORIZER_VERSION} /stackable/src/hive/hive-metastore-opa-authorizer/stackable/patches/${AUTHORIZER_VERSION} RUN <<'EOF' set -euo pipefail @@ -25,10 +25,10 @@ mkdir -p /stackable/opa-authorizer-bin # containing sources mkdir -p /stackable/opa-authorizer-src -cd "$(/stackable/patchable --images-repo-root=src checkout hive/hive-metastore-opa-authorizer ${PRODUCT_VERSION})" +cd "$(/stackable/patchable --images-repo-root=src checkout hive/hive-metastore-opa-authorizer ${AUTHORIZER_VERSION})" # Create snapshot of the source code including custom patches -tar -czf /stackable/opa-authorizer-src/hive-metastore-opa-authorizer-${PRODUCT_VERSION}-hive-${HIVE_VERSION}-hadoop-${HADOOP_VERSION}-src.tar.gz . +tar -czf /stackable/opa-authorizer-src/hive-metastore-opa-authorizer-${AUTHORIZER_VERSION}-hive-${HIVE_VERSION}-hadoop-${HADOOP_VERSION}-src.tar.gz . # The if part can be removed once we do no longer support Hive 3.x.x if [[ "${HIVE_VERSION}" =~ ^3 ]]; then diff --git a/hive/hive-metastore-opa-authorizer/boil-config.toml b/hive/hive-metastore-opa-authorizer/boil-config.toml index 32f6de722..5eff87f4d 100644 --- a/hive/hive-metastore-opa-authorizer/boil-config.toml +++ b/hive/hive-metastore-opa-authorizer/boil-config.toml @@ -1,8 +1,26 @@ -[versions."v1.0.0".local-images] +[versions."v1.0.0-hive-3.1.3-hadoop-3.3.6".local-images] "java-devel" = "11" -[versions."v1.0.0".build-arguments] +[versions."v1.0.0-hive-3.1.3-hadoop-3.3.6".build-arguments] +authorizer-version = "v1.0.0" hive-version = "3.1.3" hadoop-version = "3.3.6" +delete-caches = "true" -delete-caches = "true" \ No newline at end of file +[versions."v1.0.0-hive-4.0.1-hadoop-3.3.6".local-images] +"java-devel" = "11" + +[versions."v1.0.0-hive-4.0.1-hadoop-3.3.6".build-arguments] +authorizer-version = "v1.0.0" +hive-version = "4.0.1" +hadoop-version = "3.3.6" +delete-caches = "true" + +[versions."v1.0.0-hive-4.1.0-hadoop-3.4.2".local-images] +"java-devel" = "17" + +[versions."v1.0.0-hive-4.1.0-hadoop-3.4.2".build-arguments] +authorizer-version = "v1.0.0" +hive-version = "4.1.0" +hadoop-version = "3.4.2" +delete-caches = "true" From 3c49266138ef847b24073a8aa6c30ba22c61d9a4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 7 Nov 2025 09:08:12 +0100 Subject: [PATCH 6/7] use non-shaded opa-authz jar for 4.x.x --- hive/hive-metastore-opa-authorizer/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive/hive-metastore-opa-authorizer/Dockerfile b/hive/hive-metastore-opa-authorizer/Dockerfile index 03f3619a0..ba7db4911 100644 --- a/hive/hive-metastore-opa-authorizer/Dockerfile +++ b/hive/hive-metastore-opa-authorizer/Dockerfile @@ -36,7 +36,10 @@ if [[ "${HIVE_VERSION}" =~ ^3 ]]; then mv hms-v3/target/com.bosch.bdps.hms3-${HIVE_VERSION}-${HADOOP_VERSION}-dev.jar /stackable/opa-authorizer-bin else mvn clean package -DskipTests -Dhive.version=${HIVE_VERSION} -Dhadoop.version=${HADOOP_VERSION} -f hms-v4/pom.xml - mv hms-v4/target/com.bosch.bdps.hms4-${HIVE_VERSION}-${HADOOP_VERSION}-dev.jar /stackable/opa-authorizer-bin + + # The hive-metastore-opa-authorizer offers a shaded jar from version 4.x.x. Using the shaded jar leads to problems with schema tool at pod startup. + # mv hms-v4/target/com.bosch.bdps.hms4-${HIVE_VERSION}-${HADOOP_VERSION}-dev.jar /stackable/opa-authorizer-bin + mv hms-v4/target/hms4-dev.jar /stackable/opa-authorizer-bin fi # We're removing these to make the intermediate layer smaller From a6469f872099ad905db90b4ae1d596f510b6679a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 7 Nov 2025 12:51:02 +0100 Subject: [PATCH 7/7] adapted changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c19a20c..35405c2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- hive: Build [hive-metastore-opa-authorizer](https://github.com/boschglobal/hive-metastore-opa-authorizer) from source and add to image ([#1340]). + +[#1340]: https://github.com/stackabletech/docker-images/pull/1340 + ## [25.11.0] - 2025-11-07 ## [25.11.0-rc1] - 2025-11-06