From 2f572d24c3b51a4efb0a3beee1ea888e16ceb7b2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 13:29:39 +0200 Subject: [PATCH 01/11] chore: remove metadata storage properties --- deploy/config-spec/properties.yaml | 145 +----------------- .../druid-operator/configs/properties.yaml | 145 +----------------- 2 files changed, 16 insertions(+), 274 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 5b620e02..8289ad78 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -8,13 +8,15 @@ spec: - "/tmp/xyz" - unit: &unitPort name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ + |6553[0-5])$" - unit: &unitPrometheusNamespace name: "prometheusNamespace" regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - unit: &unitDuration name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)?)?$" + regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ + ?)?$" examples: - "PT300S" @@ -214,7 +216,6 @@ properties: comment: "TTL for host names that cannot be resolved." description: "TTL for host names that cannot be resolved." - - property: &plaintextPort propertyNames: - name: "druid.plaintextPort" @@ -418,139 +419,6 @@ properties: required: true asOfVersion: "0.0.0" - - property: &metadataStorageType - propertyNames: - - name: "druid.metadata.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "mysql" - - "postgresql" - - "derby" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageConnectURI - propertyNames: - - name: "druid.metadata.storage.connector.connectURI" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageHost - propertyNames: - - name: "druid.metadata.storage.connector.host" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStoragePort - propertyNames: - - name: "druid.metadata.storage.connector.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1024" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageUser - propertyNames: - - name: "druid.metadata.storage.connector.user" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &metadataStoragePassword - propertyNames: - - name: "druid.metadata.storage.connector.password" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - property: &indexerLogsDirectory propertyNames: - name: "druid.indexer.logs.directory" @@ -661,7 +529,10 @@ properties: type: "string" defaultValues: - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m + -Duser.timezone=UTC -Dfile.encoding=UTF-8 + -XX:+ExitOnOutOfMemoryError + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" roles: - name: "broker" required: false diff --git a/deploy/helm/druid-operator/configs/properties.yaml b/deploy/helm/druid-operator/configs/properties.yaml index 5b620e02..8289ad78 100644 --- a/deploy/helm/druid-operator/configs/properties.yaml +++ b/deploy/helm/druid-operator/configs/properties.yaml @@ -8,13 +8,15 @@ spec: - "/tmp/xyz" - unit: &unitPort name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ + |6553[0-5])$" - unit: &unitPrometheusNamespace name: "prometheusNamespace" regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - unit: &unitDuration name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)?)?$" + regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ + ?)?$" examples: - "PT300S" @@ -214,7 +216,6 @@ properties: comment: "TTL for host names that cannot be resolved." description: "TTL for host names that cannot be resolved." - - property: &plaintextPort propertyNames: - name: "druid.plaintextPort" @@ -418,139 +419,6 @@ properties: required: true asOfVersion: "0.0.0" - - property: &metadataStorageType - propertyNames: - - name: "druid.metadata.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "mysql" - - "postgresql" - - "derby" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageConnectURI - propertyNames: - - name: "druid.metadata.storage.connector.connectURI" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageHost - propertyNames: - - name: "druid.metadata.storage.connector.host" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStoragePort - propertyNames: - - name: "druid.metadata.storage.connector.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1024" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageUser - propertyNames: - - name: "druid.metadata.storage.connector.user" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &metadataStoragePassword - propertyNames: - - name: "druid.metadata.storage.connector.password" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - property: &indexerLogsDirectory propertyNames: - name: "druid.indexer.logs.directory" @@ -661,7 +529,10 @@ properties: type: "string" defaultValues: - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m + -Duser.timezone=UTC -Dfile.encoding=UTF-8 + -XX:+ExitOnOutOfMemoryError + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" roles: - name: "broker" required: false From 0ec8f9f16140fa0d151f80f40b56c029f3d61327 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 13:41:39 +0200 Subject: [PATCH 02/11] chore: smoke tests passing --- extra/crds.yaml | 136 +++++++++++++----- rust/operator-binary/src/crd/database.rs | 63 ++++++++ rust/operator-binary/src/crd/mod.rs | 82 +---------- rust/operator-binary/src/druid_controller.rs | 117 ++++++++------- rust/operator-binary/src/extensions.rs | 10 +- .../kuttl/smoke/50-install-druid.yaml.j2 | 11 +- 6 files changed, 245 insertions(+), 174 deletions(-) create mode 100644 rust/operator-binary/src/crd/database.rs diff --git a/extra/crds.yaml b/extra/crds.yaml index 959d6c31..5f7b828a 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1453,44 +1453,110 @@ spec: type: string type: object type: object - metadataStorageDatabase: + metadataDatabase: description: Druid requires an SQL database to store metadata into. Specify connection information here. + oneOf: + - required: + - postgresql + - required: + - mysql + - required: + - derby properties: - connString: + derby: description: |- - The connect string for the database, for Postgres this could look like: - `jdbc:postgresql://postgresql-druid/druid` - type: string - credentialsSecret: - description: |- - A reference to a Secret containing the database credentials. - The Secret needs to contain the keys `username` and `password`. - nullable: true - type: string - dbType: + Connection settings for an embedded [Apache Derby](https://db.apache.org/derby/) database. + + Derby is an embedded, file-based Java database engine that requires no separate server process. + It is typically used for development, testing, or as a lightweight metastore backend (e.g. for + Apache Hive). + properties: + location: + description: |- + Path on the filesystem where Derby stores its database files. + + If not specified, defaults to `/tmp/derby/{unique_database_name}/derby.db`. + The `{unique_database_name}` part is automatically handled by the operator and is added to + prevent clashing database files. The `create=true` flag is always appended to the JDBC URL, + so the database is created automatically if it does not yet exist at this location. + nullable: true + type: string + type: object + mysql: description: |- - The database type. Supported values are: `derby`, `mysql` and `postgres`. - Note that a Derby database created locally in the container is not persisted! - Derby is not suitable for production use. - enum: - - derby - - mysql - - postgresql - type: string - host: - description: The host, i.e. `postgresql-druid`. - type: string - port: - description: The port, i.e. 5432 - format: uint16 - maximum: 65535.0 - minimum: 0.0 - type: integer - required: - - connString - - dbType - - host - - port + Connection settings for a [MySQL](https://www.mysql.com/) database. + + Please note that - due to license issues - we don't ship the mysql driver, you need to add + it it yourself. + See for details. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the MySQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the MySQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 3306 + description: Port the MySQL server is listening on. Defaults to `3306`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + postgresql: + description: Connection settings for a [PostgreSQL](https://www.postgresql.org/) database. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object type: object tls: default: @@ -1529,7 +1595,7 @@ spec: type: string required: - deepStorage - - metadataStorageDatabase + - metadataDatabase - zookeeperConfigMapName type: object clusterOperation: diff --git a/rust/operator-binary/src/crd/database.rs b/rust/operator-binary/src/crd/database.rs new file mode 100644 index 00000000..305602c7 --- /dev/null +++ b/rust/operator-binary/src/crd/database.rs @@ -0,0 +1,63 @@ +use std::ops::Deref; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + database_connections::{ + databases::{ + derby::DerbyConnection, mysql::MysqlConnection, postgresql::PostgresqlConnection, + }, + drivers::jdbc::JdbcDatabaseConnection, + }, + schemars::{self, JsonSchema}, +}; + +// metadata storage config properties +pub const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; +pub const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str = + "druid.metadata.storage.connector.connectURI"; +pub const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; +pub const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum MetadataDatabaseConnection { + // Docs are on the struct + Postgresql(PostgresqlConnection), + + /// Connection settings for a [MySQL](https://www.mysql.com/) database. + /// + /// Please note that - due to license issues - we don't ship the mysql driver, you need to add + /// it it yourself. + /// See for details. + Mysql(MysqlConnection), + + // Docs are on the struct + Derby(DerbyConnection), + // We don't support generic (yet?), as we need to tell the metastore the `--dbtype` on startup, + // which is not known for generic connection. We could e.g. create a new struct with + // #[serde(flatten)] of the GenericJdbcDatabaseConnection and an additional field + // `metastoreDbType` (or similar). +} + +impl MetadataDatabaseConnection { + /// Name of the database as it should be passed using the `--db-type` CLI argument to Hive + pub fn as_db_type(&self) -> &str { + match self { + MetadataDatabaseConnection::Postgresql(_) => "postgresql", + MetadataDatabaseConnection::Mysql(_) => "mysql", + MetadataDatabaseConnection::Derby(_) => "derby", + } + } +} + +impl Deref for MetadataDatabaseConnection { + type Target = dyn JdbcDatabaseConnection; + + fn deref(&self) -> &Self::Target { + match self { + Self::Postgresql(p) => p, + Self::Mysql(m) => m, + Self::Derby(d) => d, + } + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 4c202368..ddef45b8 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -46,6 +46,7 @@ use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; use crate::crd::{ affinity::get_affinity, authorization::DruidAuthorization, + database::MetadataDatabaseConnection, resource::RoleResource, tls::{DruidTls, default_druid_tls}, }; @@ -53,6 +54,7 @@ use crate::crd::{ pub mod affinity; pub mod authentication; pub mod authorization; +pub mod database; pub mod memory; pub mod resource; pub mod security; @@ -109,21 +111,12 @@ pub const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; pub const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; pub const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; pub const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri"; -// metadata storage config properties -const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; -const METADATA_STORAGE_URI: &str = "druid.metadata.storage.connector.connectURI"; -const METADATA_STORAGE_HOST: &str = "druid.metadata.storage.connector.host"; -const METADATA_STORAGE_PORT: &str = "druid.metadata.storage.connector.port"; -const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; -const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; // indexer properties pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; // historical settings pub const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; pub const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; pub const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; -// extra -pub const CREDENTIALS_SECRET_PROPERTY: &str = "credentialsSecret"; // logs pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { value: 10.0, @@ -136,10 +129,6 @@ pub const METRICS_PORT: u16 = 9090; pub const COOKIE_PASSPHRASE_ENV: &str = "OIDC_COOKIE_PASSPHRASE"; -// DB credentials - both of these are read from an env var by Druid with the ${env:...} syntax -pub const DB_USERNAME_ENV: &str = "DB_USERNAME_ENV"; -pub const DB_PASSWORD_ENV: &str = "DB_PASSWORD_ENV"; - // Graceful shutdown timeouts const DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); const DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -361,7 +350,7 @@ pub mod versioned { pub ingestion: Option, /// Druid requires an SQL database to store metadata into. Specify connection information here. - pub metadata_storage_database: DatabaseConnectionSpec, + pub metadata_database: MetadataDatabaseConnection, /// TLS encryption settings for Druid, more information in the /// [security documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/security). @@ -420,34 +409,6 @@ impl v1alpha1::DruidCluster { match file { JVM_CONFIG => {} RUNTIME_PROPS => { - let mds = &self.spec.cluster_config.metadata_storage_database; - result.insert( - METADATA_STORAGE_TYPE.to_string(), - Some(mds.db_type.to_string()), - ); - result.insert( - METADATA_STORAGE_URI.to_string(), - Some(mds.conn_string.to_string()), - ); - result.insert( - METADATA_STORAGE_HOST.to_string(), - Some(mds.host.to_string()), - ); - result.insert( - METADATA_STORAGE_PORT.to_string(), - Some(mds.port.to_string()), - ); - if mds.credentials_secret.is_some() { - result.insert( - METADATA_STORAGE_USER.to_string(), - Some(format!("${{env:{DB_USERNAME_ENV}}}")), - ); - result.insert( - METADATA_STORAGE_PASSWORD.to_string(), - Some(format!("${{env:{DB_PASSWORD_ENV}}}")), - ); - } - // OPA if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization { @@ -1155,43 +1116,6 @@ impl DruidRole { } } -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DatabaseConnectionSpec { - /// The database type. Supported values are: `derby`, `mysql` and `postgres`. - /// Note that a Derby database created locally in the container is not persisted! - /// Derby is not suitable for production use. - pub db_type: DbType, - /// The connect string for the database, for Postgres this could look like: - /// `jdbc:postgresql://postgresql-druid/druid` - pub conn_string: String, - /// The host, i.e. `postgresql-druid`. - pub host: String, - /// The port, i.e. 5432 - pub port: u16, - /// A reference to a Secret containing the database credentials. - /// The Secret needs to contain the keys `username` and `password`. - pub credentials_secret: Option, -} - -#[derive( - Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Display, EnumString, -)] -pub enum DbType { - #[serde(rename = "derby")] - #[strum(serialize = "derby")] - #[default] - Derby, - - #[serde(rename = "mysql")] - #[strum(serialize = "mysql")] - Mysql, - - #[serde(rename = "postgresql")] - #[strum(serialize = "postgresql")] - Postgresql, -} - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize, Display)] #[serde(rename_all = "camelCase")] pub enum DeepStorageSpec { diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 819665c2..d4e974cb 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -29,7 +29,6 @@ use stackable_operator::{ opa::OpaApiVersion, product_image_selection::{self, ResolvedProductImage}, rbac::build_rbac_resources, - tls_verification::TlsClientDetailsError, }, constants::RESTART_CONTROLLER_ENABLED_LABEL, crd::s3, @@ -70,19 +69,18 @@ use crate::{ authentication::DruidAuthenticationConfig, config::jvm::construct_jvm_args, crd::{ - APP_NAME, AUTH_AUTHORIZER_OPA_URI, CREDENTIALS_SECRET_PROPERTY, CommonRoleGroupConfig, - Container, DB_PASSWORD_ENV, DB_USERNAME_ENV, DRUID_CONFIG_DIRECTORY, DS_BUCKET, - DeepStorageSpec, DruidClusterStatus, DruidRole, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, - JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, - METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, - S3_ACCESS_KEY, S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, - ZOOKEEPER_CONNECTION_STRING, authentication::AuthenticationClassesResolved, - authorization::DruidAuthorization, build_recommended_labels, build_string_list, - security::DruidTlsSecurity, v1alpha1, + APP_NAME, AUTH_AUTHORIZER_OPA_URI, CommonRoleGroupConfig, Container, + DRUID_CONFIG_DIRECTORY, DS_BUCKET, DeepStorageSpec, DruidClusterStatus, DruidRole, + EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, + LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, + OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, + S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, + authentication::AuthenticationClassesResolved, authorization::DruidAuthorization, + build_recommended_labels, build_string_list, security::DruidTlsSecurity, v1alpha1, }, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, - internal_secret::{create_shared_internal_secret, env_var_from_secret}, + internal_secret::create_shared_internal_secret, listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, group_listener_name, secret_volume_listener_scope, @@ -114,11 +112,6 @@ pub struct Ctx { #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to apply global Service"))] - ApplyRoleService { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to apply Service for {}", rolegroup))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, @@ -184,9 +177,6 @@ pub enum Error { source: stackable_operator::crd::s3::v1alpha1::ConnectionError, }, - #[snafu(display("failed to configure S3 TLS client details"))] - ConfigureS3TlsClientDetails { source: TlsClientDetailsError }, - #[snafu(display("failed to get deep storage bucket"))] GetDeepStorageBucket { source: stackable_operator::crd::s3::v1alpha1::BucketError, @@ -320,16 +310,6 @@ pub enum Error { source: crate::operations::graceful_shutdown::Error, }, - #[snafu(display("failed to build TLS certificate SecretClass Volume"))] - TlsCertSecretClassVolumeBuild { - source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, - }, - - #[snafu(display("failed to build S3 credentials SecretClass Volume"))] - S3CredentialsSecretClassVolumeBuild { - source: stackable_operator::commons::secret_class::SecretClassVolumeError, - }, - #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] AuthVolumesBuild { source: crate::authentication::Error, @@ -384,6 +364,11 @@ pub enum Error { ResolveProductImage { source: product_image_selection::Error, }, + + #[snafu(display("invalid metadata database connection"))] + InvalidMetadataDatabaseConnection { + source: stackable_operator::database_connections::Error, + }, } type Result = std::result::Result; @@ -726,6 +711,12 @@ fn build_rolegroup_config_map( })?; let role = druid.get_role(&druid_role); let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent + let metadata_database_connection_details = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("METADATA") + .context(InvalidMetadataDatabaseConnectionSnafu)?; for (property_name_kind, config) in rolegroup_config { let mut conf: BTreeMap> = Default::default(); @@ -764,6 +755,47 @@ fn build_rolegroup_config_map( ); }; + conf.insert( + crate::crd::database::METADATA_STORAGE_TYPE.to_string(), + Some( + druid + .spec + .cluster_config + .metadata_database + .as_db_type() + .to_string(), + ), + ); + + conf.insert( + crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), + Some( + metadata_database_connection_details + .connection_url + .to_string(), + ), + ); + if let Some(EnvVar { + name: username_env_name, + .. + }) = &metadata_database_connection_details.username_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_USER.to_string(), + Some(format!("${{env:{username_env_name}}}",)), + ); + } + if let Some(EnvVar { + name: password_env_name, + .. + }) = &metadata_database_connection_details.password_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), + Some(format!("${{env:{password_env_name}}}",)), + ); + } + if let Some(s3) = s3_conn { if !s3.region.is_default_config() { // Raising this as warning instead of returning an error, better safe than sorry. @@ -940,12 +972,13 @@ fn build_rolegroup_statefulset( ) .context(GracefulShutdownSnafu)?; - let credentials_secret = druid + let metadata_database_connection_details = druid .spec .cluster_config - .metadata_storage_database - .credentials_secret - .as_ref(); + .metadata_database + .jdbc_connection_details("METADATA") + .context(InvalidMetadataDatabaseConnectionSnafu)?; + let mut main_container_commands = role.main_container_prepare_commands(s3_conn); let mut prepare_container_commands = vec![]; if let Some(ContainerLogConfig { @@ -1030,12 +1063,13 @@ fn build_rolegroup_statefulset( .build(), ); + metadata_database_connection_details.add_to_container(&mut cb_druid); + // rest of env let mut rest_env = rolegroup_config .get(&PropertyNameKind::Env) .iter() .flat_map(|env_vars| env_vars.iter()) - .filter(|(k, _)| k != &&CREDENTIALS_SECRET_PROPERTY.to_string()) .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()), @@ -1043,21 +1077,6 @@ fn build_rolegroup_statefulset( }) .collect::>(); - // load database credentials to environment variables: these will be used to replace - // the placeholders in runtime.properties so that the operator does not "touch" the secret. - if let Some(credentials_secret_name) = credentials_secret { - rest_env.push(env_var_from_secret( - credentials_secret_name, - Some("username"), - DB_USERNAME_ENV, - )); - rest_env.push(env_var_from_secret( - credentials_secret_name, - Some("password"), - DB_PASSWORD_ENV, - )); - } - if let Some(auth_config) = druid_auth_config { rest_env.extend(auth_config.get_env_var_mounts(druid, role)) } diff --git a/rust/operator-binary/src/extensions.rs b/rust/operator-binary/src/extensions.rs index 209a3948..8b39752f 100644 --- a/rust/operator-binary/src/extensions.rs +++ b/rust/operator-binary/src/extensions.rs @@ -4,7 +4,7 @@ use tracing::debug; use crate::{ authentication::DruidAuthenticationConfig, - crd::{DbType, security::DruidTlsSecurity, v1alpha1}, + crd::{database::MetadataDatabaseConnection, security::DruidTlsSecurity, v1alpha1}, }; const EXT_S3: &str = "druid-s3-extensions"; @@ -33,12 +33,12 @@ pub fn get_extension_list( EXT_HDFS.to_string(), ]); - match druid.spec.cluster_config.metadata_storage_database.db_type { - DbType::Derby => {} // no additional extensions required - DbType::Postgresql => { + match druid.spec.cluster_config.metadata_database { + MetadataDatabaseConnection::Derby(..) => {} // no additional extensions required + MetadataDatabaseConnection::Postgresql(..) => { extensions.insert(EXT_PSQL_MD_ST.to_string()); } - DbType::Mysql => { + MetadataDatabaseConnection::Mysql(..) => { extensions.insert(EXT_MYSQL_MD_ST.to_string()); } }; diff --git a/tests/templates/kuttl/smoke/50-install-druid.yaml.j2 b/tests/templates/kuttl/smoke/50-install-druid.yaml.j2 index 73f2c4bd..939a8170 100644 --- a/tests/templates/kuttl/smoke/50-install-druid.yaml.j2 +++ b/tests/templates/kuttl/smoke/50-install-druid.yaml.j2 @@ -23,12 +23,11 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: druid-credentials + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: druid-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From 6c8d1547e827db74c8983fc1d412095f0a5a7988 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 13:47:32 +0200 Subject: [PATCH 03/11] chore: fix hive related comment --- rust/operator-binary/src/crd/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/database.rs b/rust/operator-binary/src/crd/database.rs index 305602c7..0d96dcff 100644 --- a/rust/operator-binary/src/crd/database.rs +++ b/rust/operator-binary/src/crd/database.rs @@ -40,7 +40,7 @@ pub enum MetadataDatabaseConnection { } impl MetadataDatabaseConnection { - /// Name of the database as it should be passed using the `--db-type` CLI argument to Hive + /// Name of the database as it should be passed using `METADATA_STORAGE_TYPE` property. pub fn as_db_type(&self) -> &str { match self { MetadataDatabaseConnection::Postgresql(_) => "postgresql", From 41ffc8f9d45167581f49fb34e2cd249911103cd5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 13:48:06 +0200 Subject: [PATCH 04/11] chore: adapt changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68fbd577..24775dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,19 @@ All notable changes to this project will be documented in this file. ### Changed - BREAKING: `configOverrides` now only accepts the supported config file names `runtime.properties`, `jvm.config` and `security.properties`. Previously arbitrary keys were silently accepted but ignored ([#813]). +- BREAKING: Implement generic database connection for metadata storage ([#814]). + Renamed CRD: `metadataStorageDatabase` -> `metadataDatabase`. + The `metadataDatabase` has subfields according to the supported db types: `postgresql`, `mysql` and `derby`. - Bump `stackable-operator` to 0.110.1 and `kube` to 3.1.0 ([#813]). - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#810]). +### Deleted + +- Removed all metadata storage related properties from product config ([#814]). + [#810]: https://github.com/stackabletech/druid-operator/pull/810 [#813]: https://github.com/stackabletech/druid-operator/pull/813 +[#814]: https://github.com/stackabletech/druid-operator/pull/814 ## [26.3.0] - 2026-03-16 From 4ab2fc2fc2ac7588dc093ac4822dcaf95d8226d5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 14:07:21 +0200 Subject: [PATCH 05/11] fix: adapt unit tests to new metadatabase crd --- rust/operator-binary/src/config/jvm.rs | 22 ++- rust/operator-binary/src/crd/affinity.rs | 11 +- .../operator-binary/src/crd/authentication.rs | 8 +- rust/operator-binary/src/crd/tls.rs | 8 +- .../crd/resource_merge/druid_cluster.yaml | 11 +- .../crd/resource_merge/segment_cache.yaml | 11 +- .../crd/role_service/druid_cluster.yaml | 11 +- .../druid_controller/properties.yaml | 144 +----------------- .../druid_controller/segment_cache.yaml | 11 +- .../resources/druid_controller/simple.yaml | 11 +- 10 files changed, 54 insertions(+), 194 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 86b88be1..90f8ffa0 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -100,12 +100,11 @@ mod tests { hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: simple-druid-znode brokers: roleGroups: @@ -187,12 +186,11 @@ mod tests { hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: simple-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index e3d882df..23b61397 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -112,12 +112,11 @@ mod tests { hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: simple-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 1d1f1088..98374f33 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -258,11 +258,9 @@ deepStorage: hdfs: configMapName: druid-hdfs directory: /druid -metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 +metadataDatabase: + derby: + location: localhost:1527/var/druid/metadata.db;create=true zookeeperConfigMapName: zk-config-map "#; diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index b4fc25b3..89ec6ad9 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -40,11 +40,9 @@ deepStorage: hdfs: configMapName: druid-hdfs directory: /druid -metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 +metadataDatabase: + derby: + location: localhost:1527/var/druid/metadata.db;create=true zookeeperConfigMapName: zk-config-map "#; diff --git a/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml index 8b75a020..b0811a90 100644 --- a/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml +++ b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml @@ -13,12 +13,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: psql-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml index adc53f2d..bab1a998 100644 --- a/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml +++ b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml @@ -13,12 +13,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: psql-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml index ffb9a6a9..507870c1 100644 --- a/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml +++ b/rust/operator-binary/test/resources/crd/role_service/druid_cluster.yaml @@ -13,12 +13,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: psql-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/test/resources/druid_controller/properties.yaml b/rust/operator-binary/test/resources/druid_controller/properties.yaml index 2fb07f49..e1e167d9 100644 --- a/rust/operator-binary/test/resources/druid_controller/properties.yaml +++ b/rust/operator-binary/test/resources/druid_controller/properties.yaml @@ -9,13 +9,15 @@ spec: - "/tmp/xyz" - unit: &unitPort name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ + |6553[0-5])$" - unit: &unitPrometheusNamespace name: "prometheusNamespace" regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - unit: &unitDuration name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)?)?$" + regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ + ?)?$" examples: - "PT300S" @@ -229,139 +231,6 @@ properties: required: true asOfVersion: "0.0.0" - - property: &metadataStorageType - propertyNames: - - name: "druid.metadata.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "mysql" - - "postgresql" - - "derby" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageConnectURI - propertyNames: - - name: "druid.metadata.storage.connector.connectURI" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageHost - propertyNames: - - name: "druid.metadata.storage.connector.host" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStoragePort - propertyNames: - - name: "druid.metadata.storage.connector.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1024" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &metadataStorageUser - propertyNames: - - name: "druid.metadata.storage.connector.user" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &metadataStoragePassword - propertyNames: - - name: "druid.metadata.storage.connector.password" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - property: &indexerLogsDirectory propertyNames: - name: "druid.indexer.logs.directory" @@ -472,7 +341,10 @@ properties: type: "string" defaultValues: - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m + -Duser.timezone=UTC -Dfile.encoding=UTF-8 + -XX:+ExitOnOutOfMemoryError + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" roles: - name: "broker" required: false diff --git a/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml b/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml index dde8c761..ddb2811b 100644 --- a/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml +++ b/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml @@ -21,12 +21,11 @@ spec: accessStyle: Path credentials: secretClass: druid-s3-credentials - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: psql-druid-znode brokers: roleGroups: diff --git a/rust/operator-binary/test/resources/druid_controller/simple.yaml b/rust/operator-binary/test/resources/druid_controller/simple.yaml index 672bb2a2..bd4983ac 100644 --- a/rust/operator-binary/test/resources/druid_controller/simple.yaml +++ b/rust/operator-binary/test/resources/druid_controller/simple.yaml @@ -13,12 +13,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: mySecret + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret zookeeperConfigMapName: psql-druid-znode additionalExtensions: - druid-avro-extensions From 6a31aefff15e18a99524bf1e9a565cdb15a9b412 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 14:20:08 +0200 Subject: [PATCH 06/11] chore: replace remaining metadataStorageDatabase occurances. --- .../modules/druid/examples/getting_started/druid.yaml | 11 +++++------ .../druid/examples/getting_started/druid.yaml.j2 | 11 +++++------ rust/operator-binary/src/crd/authentication.rs | 3 +-- rust/operator-binary/src/crd/tls.rs | 3 +-- .../kuttl/authorizer/04-install-druid.yaml.j2 | 7 ++----- .../kuttl/cluster-operation/30-install-druid.yaml.j2 | 7 ++----- .../kuttl/cluster-operation/40-stop-druid.yaml.j2 | 7 ++----- .../kuttl/cluster-operation/50-pause-druid.yaml.j2 | 7 ++----- .../kuttl/cluster-operation/60-restart-druid.yaml.j2 | 7 ++----- .../templates/kuttl/external-access/50_druid.yaml.j2 | 11 +++++------ .../kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 | 7 ++----- .../ingestion-no-s3-ext/03-install-druid.yaml.j2 | 7 ++----- .../kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 | 7 ++----- tests/templates/kuttl/ldap/13-install-druid.yaml.j2 | 7 ++----- .../templates/kuttl/logging/05-install-druid.yaml.j2 | 7 ++----- tests/templates/kuttl/oidc/40_druid.yaml.j2 | 7 ++----- .../kuttl/orphaned-resources/03-install-druid.yaml.j2 | 7 ++----- .../kuttl/orphaned-resources/04-update-druid.yaml.j2 | 7 ++----- .../kuttl/overrides/20-install-druid.yaml.j2 | 7 ++----- .../kuttl/resources/30-install-druid.yaml.j2 | 7 ++----- .../kuttl/s3-deep-storage/10-install-druid.yaml.j2 | 7 ++----- tests/templates/kuttl/tls/04-install-druid.yaml.j2 | 7 ++----- 22 files changed, 51 insertions(+), 107 deletions(-) diff --git a/docs/modules/druid/examples/getting_started/druid.yaml b/docs/modules/druid/examples/getting_started/druid.yaml index 7106458b..7da5cf42 100644 --- a/docs/modules/druid/examples/getting_started/druid.yaml +++ b/docs/modules/druid/examples/getting_started/druid.yaml @@ -12,12 +12,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://postgresql-druid/druid - host: postgresql-druid - port: 5432 - credentialsSecret: druid-db-credentials + metadataDatabase: + postgresql: + host: postgresql-druid + database: druid + credentialsSecretName: druid-db-credentials brokers: roleGroups: default: diff --git a/docs/modules/druid/examples/getting_started/druid.yaml.j2 b/docs/modules/druid/examples/getting_started/druid.yaml.j2 index 7106458b..7da5cf42 100644 --- a/docs/modules/druid/examples/getting_started/druid.yaml.j2 +++ b/docs/modules/druid/examples/getting_started/druid.yaml.j2 @@ -12,12 +12,11 @@ spec: hdfs: configMapName: simple-hdfs directory: /druid - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://postgresql-druid/druid - host: postgresql-druid - port: 5432 - credentialsSecret: druid-db-credentials + metadataDatabase: + postgresql: + host: postgresql-druid + database: druid + credentialsSecretName: druid-db-credentials brokers: roleGroups: default: diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 98374f33..034723e5 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -259,8 +259,7 @@ deepStorage: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: - location: localhost:1527/var/druid/metadata.db;create=true + derby: {} zookeeperConfigMapName: zk-config-map "#; diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index 89ec6ad9..c8011b22 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -41,8 +41,7 @@ deepStorage: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: - location: localhost:1527/var/druid/metadata.db;create=true + derby: {} zookeeperConfigMapName: zk-config-map "#; diff --git a/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 b/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 index 3ca3911a..764bd2c5 100644 --- a/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 +++ b/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 @@ -27,11 +27,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 index 96fe77cf..78d0e554 100644 --- a/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 @@ -21,11 +21,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 index 92a03b8c..42beb115 100644 --- a/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 @@ -20,11 +20,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 index 420e199c..84b1ead8 100644 --- a/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 @@ -20,11 +20,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 index a40b727e..5053c55f 100644 --- a/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 @@ -20,11 +20,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/external-access/50_druid.yaml.j2 b/tests/templates/kuttl/external-access/50_druid.yaml.j2 index 904b6001..fed375fe 100644 --- a/tests/templates/kuttl/external-access/50_druid.yaml.j2 +++ b/tests/templates/kuttl/external-access/50_druid.yaml.j2 @@ -13,12 +13,11 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - metadataStorageDatabase: - dbType: postgresql - connString: jdbc:postgresql://druid-postgresql/druid - host: druid-postgresql - port: 5432 - credentialsSecret: druid-credentials + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: druid-credentials deepStorage: hdfs: configMapName: druid-hdfs diff --git a/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 b/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 index 17842f7b..dcca5636 100644 --- a/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 @@ -23,11 +23,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 b/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 index 32913f02..c71310e7 100644 --- a/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 @@ -23,11 +23,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 b/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 index 048c34f1..e12bed4e 100644 --- a/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 @@ -29,11 +29,8 @@ spec: host: s3-de-central.profitbricks.com credentials: secretClass: s3-credentials-class - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 index f172afd4..3d0532ec 100644 --- a/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 @@ -27,11 +27,8 @@ commands: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} zookeeperConfigMapName: druid-znode {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/logging/05-install-druid.yaml.j2 b/tests/templates/kuttl/logging/05-install-druid.yaml.j2 index 859c1e98..38a2fc1f 100644 --- a/tests/templates/kuttl/logging/05-install-druid.yaml.j2 +++ b/tests/templates/kuttl/logging/05-install-druid.yaml.j2 @@ -40,11 +40,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} vectorAggregatorConfigMapName: druid-vector-aggregator-discovery zookeeperConfigMapName: druid-znode brokers: diff --git a/tests/templates/kuttl/oidc/40_druid.yaml.j2 b/tests/templates/kuttl/oidc/40_druid.yaml.j2 index f7ebabe5..2b85a0ee 100644 --- a/tests/templates/kuttl/oidc/40_druid.yaml.j2 +++ b/tests/templates/kuttl/oidc/40_druid.yaml.j2 @@ -58,11 +58,8 @@ spec: secretClass: minio-tls-certificates {% endif %} baseKey: deep-storage - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 b/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 index 32913f02..c71310e7 100644 --- a/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 @@ -23,11 +23,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 b/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 index f7753cd8..39a7aea0 100644 --- a/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 @@ -23,11 +23,8 @@ spec: hdfs: configMapName: druid-hdfs directory: /druid - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 b/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 index 8f6e45c3..28a76f9d 100644 --- a/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 +++ b/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 @@ -17,11 +17,8 @@ spec: hdfs: configMapName: hdfs directory: /data - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} tls: null zookeeperConfigMapName: druid-znode brokers: diff --git a/tests/templates/kuttl/resources/30-install-druid.yaml.j2 b/tests/templates/kuttl/resources/30-install-druid.yaml.j2 index 9a6d7778..0537a2b5 100644 --- a/tests/templates/kuttl/resources/30-install-druid.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-druid.yaml.j2 @@ -27,11 +27,8 @@ spec: host: minio-druid port: 9000 accessStyle: Path - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 b/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 index e2399ddc..00cc99b4 100644 --- a/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 +++ b/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 @@ -59,11 +59,8 @@ spec: secretClass: minio-tls-certificates {% endif %} baseKey: deep-storage - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/tls/04-install-druid.yaml.j2 b/tests/templates/kuttl/tls/04-install-druid.yaml.j2 index 5f0e68a7..d754c1d7 100644 --- a/tests/templates/kuttl/tls/04-install-druid.yaml.j2 +++ b/tests/templates/kuttl/tls/04-install-druid.yaml.j2 @@ -83,11 +83,8 @@ spec: caCert: secretClass: minio-tls-certificates baseKey: deep-storage - metadataStorageDatabase: - dbType: derby - connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true - host: localhost - port: 1527 + metadataDatabase: + derby: {} {% if test_scenario['values']['tls-mode'] == 'internal-and-server-tls' or test_scenario['values']['tls-mode'] == 'internal-and-server-tls-and-tls-client-auth' %} tls: serverAndInternalSecretClass: druid-tls From 0cd6ccb2c0d8768304260b37043c3813954bf702 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 15:08:26 +0200 Subject: [PATCH 07/11] fix: use derby references to network client --- rust/operator-binary/src/crd/authentication.rs | 3 ++- rust/operator-binary/src/crd/tls.rs | 3 ++- rust/operator-binary/src/druid_controller.rs | 6 ++++-- .../kuttl/cluster-operation/30-install-druid.yaml.j2 | 3 ++- .../templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 | 3 ++- .../kuttl/cluster-operation/50-pause-druid.yaml.j2 | 3 ++- .../kuttl/cluster-operation/60-restart-druid.yaml.j2 | 3 ++- .../kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 | 3 ++- .../kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 | 3 ++- .../kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 | 3 ++- tests/templates/kuttl/ldap/13-install-druid.yaml.j2 | 3 ++- tests/templates/kuttl/logging/05-install-druid.yaml.j2 | 3 ++- tests/templates/kuttl/oidc/40_druid.yaml.j2 | 3 ++- .../kuttl/orphaned-resources/03-install-druid.yaml.j2 | 3 ++- .../kuttl/orphaned-resources/04-update-druid.yaml.j2 | 3 ++- tests/templates/kuttl/overrides/20-install-druid.yaml.j2 | 3 ++- tests/templates/kuttl/resources/30-install-druid.yaml.j2 | 3 ++- .../kuttl/s3-deep-storage/10-install-druid.yaml.j2 | 3 ++- tests/templates/kuttl/tls/04-install-druid.yaml.j2 | 3 ++- 19 files changed, 40 insertions(+), 20 deletions(-) diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 034723e5..8f03f21c 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -259,7 +259,8 @@ deepStorage: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" zookeeperConfigMapName: zk-config-map "#; diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index c8011b22..9020da47 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -41,7 +41,8 @@ deepStorage: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" zookeeperConfigMapName: zk-config-map "#; diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index d4e974cb..177d2b9c 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -715,7 +715,7 @@ fn build_rolegroup_config_map( .spec .cluster_config .metadata_database - .jdbc_connection_details("METADATA") + .jdbc_connection_details("metadata") .context(InvalidMetadataDatabaseConnectionSnafu)?; for (property_name_kind, config) in rolegroup_config { @@ -775,6 +775,7 @@ fn build_rolegroup_config_map( .to_string(), ), ); + if let Some(EnvVar { name: username_env_name, .. @@ -785,6 +786,7 @@ fn build_rolegroup_config_map( Some(format!("${{env:{username_env_name}}}",)), ); } + if let Some(EnvVar { name: password_env_name, .. @@ -976,7 +978,7 @@ fn build_rolegroup_statefulset( .spec .cluster_config .metadata_database - .jdbc_connection_details("METADATA") + .jdbc_connection_details("metadata") .context(InvalidMetadataDatabaseConnectionSnafu)?; let mut main_container_commands = role.main_container_prepare_commands(s3_conn); diff --git a/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 index 78d0e554..67d1255e 100644 --- a/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-install-druid.yaml.j2 @@ -22,7 +22,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 index 42beb115..48d141f3 100644 --- a/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/40-stop-druid.yaml.j2 @@ -21,7 +21,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 index 84b1ead8..4cd27afc 100644 --- a/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/50-pause-druid.yaml.j2 @@ -21,7 +21,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 b/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 index 5053c55f..9778c1e1 100644 --- a/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/60-restart-druid.yaml.j2 @@ -21,7 +21,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 b/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 index dcca5636..f7eb5fae 100644 --- a/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/hdfs-deep-storage/03-install-druid.yaml.j2 @@ -24,7 +24,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 b/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 index c71310e7..f66b2476 100644 --- a/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ingestion-no-s3-ext/03-install-druid.yaml.j2 @@ -24,7 +24,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 b/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 index e12bed4e..519adbb0 100644 --- a/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ingestion-s3-ext/03-install-druid.yaml.j2 @@ -30,7 +30,8 @@ spec: credentials: secretClass: s3-credentials-class metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 index 3d0532ec..f1bce875 100644 --- a/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 +++ b/tests/templates/kuttl/ldap/13-install-druid.yaml.j2 @@ -28,7 +28,8 @@ commands: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" zookeeperConfigMapName: druid-znode {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/logging/05-install-druid.yaml.j2 b/tests/templates/kuttl/logging/05-install-druid.yaml.j2 index 38a2fc1f..b8727130 100644 --- a/tests/templates/kuttl/logging/05-install-druid.yaml.j2 +++ b/tests/templates/kuttl/logging/05-install-druid.yaml.j2 @@ -41,7 +41,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" vectorAggregatorConfigMapName: druid-vector-aggregator-discovery zookeeperConfigMapName: druid-znode brokers: diff --git a/tests/templates/kuttl/oidc/40_druid.yaml.j2 b/tests/templates/kuttl/oidc/40_druid.yaml.j2 index 2b85a0ee..7406ae05 100644 --- a/tests/templates/kuttl/oidc/40_druid.yaml.j2 +++ b/tests/templates/kuttl/oidc/40_druid.yaml.j2 @@ -59,7 +59,8 @@ spec: {% endif %} baseKey: deep-storage metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 b/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 index c71310e7..f66b2476 100644 --- a/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/03-install-druid.yaml.j2 @@ -24,7 +24,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 b/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 index 39a7aea0..d27dbf14 100644 --- a/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/04-update-druid.yaml.j2 @@ -24,7 +24,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 b/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 index 28a76f9d..c2f518b2 100644 --- a/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 +++ b/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 @@ -18,7 +18,8 @@ spec: configMapName: hdfs directory: /data metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" tls: null zookeeperConfigMapName: druid-znode brokers: diff --git a/tests/templates/kuttl/resources/30-install-druid.yaml.j2 b/tests/templates/kuttl/resources/30-install-druid.yaml.j2 index 0537a2b5..f846b5d2 100644 --- a/tests/templates/kuttl/resources/30-install-druid.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-druid.yaml.j2 @@ -28,7 +28,8 @@ spec: port: 9000 accessStyle: Path metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 b/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 index 00cc99b4..53e1ba10 100644 --- a/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 +++ b/tests/templates/kuttl/s3-deep-storage/10-install-druid.yaml.j2 @@ -60,7 +60,8 @@ spec: {% endif %} baseKey: deep-storage metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/tls/04-install-druid.yaml.j2 b/tests/templates/kuttl/tls/04-install-druid.yaml.j2 index d754c1d7..ccd87e5d 100644 --- a/tests/templates/kuttl/tls/04-install-druid.yaml.j2 +++ b/tests/templates/kuttl/tls/04-install-druid.yaml.j2 @@ -84,7 +84,8 @@ spec: secretClass: minio-tls-certificates baseKey: deep-storage metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if test_scenario['values']['tls-mode'] == 'internal-and-server-tls' or test_scenario['values']['tls-mode'] == 'internal-and-server-tls-and-tls-client-auth' %} tls: serverAndInternalSecretClass: druid-tls From 5f3cefcc3247772f5bcd57be972f9f51bbb4b109 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 15:08:48 +0200 Subject: [PATCH 08/11] fix: remove derby warning from getting started (uses postgres) --- docs/modules/druid/pages/getting_started/first_steps.adoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/modules/druid/pages/getting_started/first_steps.adoc b/docs/modules/druid/pages/getting_started/first_steps.adoc index 7a7db01a..f4c8cf7b 100644 --- a/docs/modules/druid/pages/getting_started/first_steps.adoc +++ b/docs/modules/druid/pages/getting_started/first_steps.adoc @@ -76,10 +76,6 @@ include::example$getting_started/getting_started.sh[tag=install-druid] This creates the actual Druid Stacklet. -WARNING: This Druid instance uses Derby (`dbType: derby`) as a metadata store, which is an interal SQL database. -It is not persisted and not suitable for production use! -Consult the https://druid.apache.org/docs/latest/dependencies/metadata-storage.html#available-metadata-stores[Druid documentation] for a list of supported databases and setup instructions for production instances. - == Verify that it works Submit an ingestion job and then query the ingested data -- either through the web interface or the API. From 56fedd2e046f7da4fed22ca6dba7a4afc1c2234d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 15:26:20 +0200 Subject: [PATCH 09/11] fix: yaml lints --- deploy/config-spec/properties.yaml | 1 + deploy/helm/chart_testing.yaml | 1 + .../druid-operator/configs/properties.yaml | 1 + dev-cluster.yml | 21 ++++++++++--------- tests/interu.yaml | 1 + 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 8289ad78..e4ede43b 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,3 +1,4 @@ +--- version: 0.1.0 spec: units: diff --git a/deploy/helm/chart_testing.yaml b/deploy/helm/chart_testing.yaml index 82b39c26..253af46d 100644 --- a/deploy/helm/chart_testing.yaml +++ b/deploy/helm/chart_testing.yaml @@ -1,3 +1,4 @@ +--- remote: origin target-branch: main chart-dirs: diff --git a/deploy/helm/druid-operator/configs/properties.yaml b/deploy/helm/druid-operator/configs/properties.yaml index 8289ad78..e4ede43b 100644 --- a/deploy/helm/druid-operator/configs/properties.yaml +++ b/deploy/helm/druid-operator/configs/properties.yaml @@ -1,3 +1,4 @@ +--- version: 0.1.0 spec: units: diff --git a/dev-cluster.yml b/dev-cluster.yml index 01279c6f..513d9242 100644 --- a/dev-cluster.yml +++ b/dev-cluster.yml @@ -1,3 +1,4 @@ +--- # # Create a cluster: # @@ -10,13 +11,13 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane -- role: worker - kubeadmConfigPatches: - - | - kind: JoinConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "nodeType=druid-data" -- role: worker -- role: worker + - role: control-plane + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "nodeType=druid-data" + - role: worker + - role: worker diff --git a/tests/interu.yaml b/tests/interu.yaml index f1835759..56bafbdd 100644 --- a/tests/interu.yaml +++ b/tests/interu.yaml @@ -1,3 +1,4 @@ +--- runners: amd64: platform: aks-1.32 From 37e2dd9e093f7ff1d22e441cafd934586dcb3fea Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 15:28:03 +0200 Subject: [PATCH 10/11] fix: missing derby change in test --- tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 b/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 index 764bd2c5..9d120072 100644 --- a/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 +++ b/tests/templates/kuttl/authorizer/04-install-druid.yaml.j2 @@ -28,7 +28,8 @@ spec: configMapName: druid-hdfs directory: /druid metadataDatabase: - derby: {} + derby: + location: "//localhost:1527/var/druid/metadata.db" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From c504c3bfb81d2d91c033967e47eca589e873ab82 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 21 Apr 2026 16:01:14 +0200 Subject: [PATCH 11/11] fix: remove c&p hive references enum / method comments --- extra/crds.yaml | 1 - rust/operator-binary/src/crd/database.rs | 9 +++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 5f7b828a..0e61f44f 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1488,7 +1488,6 @@ spec: Please note that - due to license issues - we don't ship the mysql driver, you need to add it it yourself. - See for details. properties: credentialsSecretName: description: |- diff --git a/rust/operator-binary/src/crd/database.rs b/rust/operator-binary/src/crd/database.rs index 0d96dcff..43f64078 100644 --- a/rust/operator-binary/src/crd/database.rs +++ b/rust/operator-binary/src/crd/database.rs @@ -28,19 +28,16 @@ pub enum MetadataDatabaseConnection { /// /// Please note that - due to license issues - we don't ship the mysql driver, you need to add /// it it yourself. - /// See for details. Mysql(MysqlConnection), // Docs are on the struct Derby(DerbyConnection), - // We don't support generic (yet?), as we need to tell the metastore the `--dbtype` on startup, - // which is not known for generic connection. We could e.g. create a new struct with - // #[serde(flatten)] of the GenericJdbcDatabaseConnection and an additional field - // `metastoreDbType` (or similar). + // We don't support generic as druid only offers the types mentioned above for metadata storage + // See } impl MetadataDatabaseConnection { - /// Name of the database as it should be passed using `METADATA_STORAGE_TYPE` property. + /// Name of the database as it should be passed using `METADATA_STORAGE_TYPE` ("druid.metadata.storage.type") property. pub fn as_db_type(&self) -> &str { match self { MetadataDatabaseConnection::Postgresql(_) => "postgresql",