From 94a0d96683b2588d836e1a2b6b8212b35eb40407 Mon Sep 17 00:00:00 2001 From: ramk Date: Tue, 23 Jun 2026 19:01:04 +0530 Subject: [PATCH 01/10] RANGER-5655: Implement dynamic unified ingestor registry for audit-ingestor: runtime Kafka partition routing and per-repo service allowlists via compacted topic + REST, without ingestor restarts. Feature flag default off. Co-authored-by: Cursor --- .../audit/server/AuditServerConstants.java | 16 +- .../audit/utils/AuditMessageQueueUtils.java | 159 +++++-- .../utils/AuditMessageQueueUtilsTest.java | 13 + .../producer/kafka/AuditMessageQueue.java | 30 ++ .../producer/kafka/AuditPartitioner.java | 186 +++++++- .../partition/AuthToLocalRuleCatalog.java | 158 +++++++ .../partition/AuthToLocalRuleComposer.java | 216 ++++++++++ .../KafkaAuditTopicPartitionGrower.java | 32 ++ .../partition/KafkaPartitionPlanRegistry.java | 124 ++++++ .../partition/PartitionPlanAllocator.java | 199 +++++++++ .../partition/PartitionPlanBootstrap.java | 111 +++++ .../PartitionPlanBootstrapConfig.java | 131 ++++++ .../kafka/partition/PartitionPlanHolder.java | 90 ++++ .../partition/PartitionPlanKafkaConfig.java | 124 ++++++ .../partition/PartitionPlanRegistry.java | 32 ++ .../PartitionPlanRegistryFactory.java | 30 ++ .../PartitionPlanRequestValidator.java | 121 ++++++ .../kafka/partition/PartitionPlanService.java | 225 ++++++++++ .../partition/PartitionPlanUpdateApplier.java | 70 +++ .../partition/PartitionPlanValidator.java | 123 ++++++ .../kafka/partition/PartitionPlanWatcher.java | 175 ++++++++ .../kafka/partition/PrimaryCatalogRule.java | 31 ++ .../partition/ServiceAllowlistBootstrap.java | 105 +++++ .../partition/ServiceAllowlistResolver.java | 62 +++ .../constants/PartitionPlanConstants.java | 33 ++ .../PartitionPlanConflictException.java | 38 ++ .../exception/PartitionPlanException.java | 35 ++ .../kafka/partition/model/OnboardService.java | 69 +++ .../kafka/partition/model/PartitionPlan.java | 251 +++++++++++ .../model/PartitionPlanReplacement.java | 195 +++++++++ .../model/PluginPartitionAssignment.java | 97 +++++ .../kafka/partition/model/PluginScale.java | 49 +++ .../kafka/partition/model/PromotePlugin.java | 73 ++++ .../model/ServiceAllowlistEntry.java | 105 +++++ .../apache/ranger/audit/rest/AuditREST.java | 286 ++++++++++++- .../audit/server/AuditServerConfig.java | 10 +- .../conf/ranger-audit-ingestor-site.xml | 101 ++++- .../kafka/AuditPartitionerDynamicTest.java | 187 ++++++++ .../AuthToLocalRuleComposerTest.java | 218 ++++++++++ .../partition/PartitionPlanAllocatorTest.java | 111 +++++ .../PartitionPlanBootstrapSupportTest.java | 142 +++++++ .../partition/PartitionPlanBootstrapTest.java | 84 ++++ .../partition/PartitionPlanHolderTest.java | 100 +++++ .../PartitionPlanKafkaConfigTest.java | 73 ++++ .../PartitionPlanReplacementTest.java | 154 +++++++ .../PartitionPlanServiceMutationTest.java | 399 ++++++++++++++++++ .../partition/PartitionPlanServiceTest.java | 95 +++++ .../PartitionPlanUpdateApplierTest.java | 110 +++++ .../partition/PartitionPlanValidatorTest.java | 70 +++ .../ServiceAllowlistBootstrapTest.java | 101 +++++ .../ServiceAllowlistResolverTest.java | 96 +++++ .../model/PartitionPlanJsonTest.java | 64 +++ 52 files changed, 5844 insertions(+), 65 deletions(-) create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index 58b136b8512..cbbdf591ba7 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -62,6 +62,18 @@ private AuditServerConstants() {} public static final String PROP_BUFFER_PARTITIONS = "kafka.topic.partitions.buffer"; public static final String PROP_REPLICATION_FACTOR = "kafka.replication.factor"; + // Dynamic partition plan (Kafka compacted registry topic) + public static final String PROP_PARTITION_PLAN_TOPIC = "kafka.partition.plan.topic"; + public static final String PROP_PARTITION_PLAN_REFRESH_INTERVAL_MS = "kafka.partition.plan.refresh.interval.ms"; + public static final String PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS = "kafka.partition.plan.consumer.poll.timeout.ms"; + public static final String PROP_PARTITION_PLAN_DYNAMIC_ENABLED = "kafka.partition.plan.dynamic.enabled"; + public static final String PROP_PARTITION_PLAN_ALLOWED_USERS = "kafka.partition.plan.allowed.users"; + public static final String DEFAULT_PARTITION_PLAN_TOPIC = "ranger_audit_partition_plan"; + public static final int DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS = 30000; + public static final int DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS = 500; + public static final int PARTITION_PLAN_TOPIC_PARTITION_COUNT = 1; + public static final String KAFKA_TOPIC_CLEANUP_POLICY_COMPACT = "compact"; + // Kafka producer tuning (ranger.audit.ingestor.kafka.producer.*) public static final String PROP_KAFKA_PRODUCER_PREFIX = "kafka.producer."; public static final String PROP_PRODUCER_BATCH_SIZE = "batch.size"; @@ -108,9 +120,9 @@ private AuditServerConstants() {} public static final long DEFAULT_OFFSET_COMMIT_INTERVAL_MS = 30000; // 30 seconds public static final int DEFAULT_MAX_POLL_RECORDS = 500; // Kafka default batch size - // Default configured plugins: each gets allocated partitions from the topic + // Empty by default: operators opt in via XML (static) or REST (dynamic). See ranger-audit-ingestor-site.xml. public static final String DEFAULT_PARTITIONER_CLASS = "org.apache.ranger.audit.producer.kafka.AuditPartitioner"; - public static final String DEFAULT_CONFIGURED_PLUGINS = "hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi"; + public static final String DEFAULT_CONFIGURED_PLUGINS = ""; public static final short DEFAULT_REPLICATION_FACTOR = 3; public static final int DEFAULT_TOPIC_PARTITIONS = 10; public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3; diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java index bfba27260ae..f1e83125760 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java @@ -27,6 +27,7 @@ import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.errors.TopicExistsException; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConstants; import org.slf4j.Logger; @@ -39,6 +40,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ExecutionException; public class AuditMessageQueueUtils { private static final Logger LOG = LoggerFactory.getLogger(AuditMessageQueueUtils.class); @@ -50,30 +52,13 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP LOG.info("==> AuditMessageQueueUtils:createAuditsTopicIfNotExists(propPrefix={})", propPrefix); String ret = null; - String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); - String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); - String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); - String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM); - int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000); - int partitions = getPartitions(props, propPrefix); - short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); - int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000); - int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; - int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); + String topicName = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + int partitions = getPartitions(props, propPrefix); + short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); + int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; + int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); - Map kafkaProp = new HashMap<>(); - - kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - kafkaProp.put("sasl.mechanism", saslMechanism); - kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); - - if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) { - kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix)); - kafkaProp.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); - } - - kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS); - kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS); + Map kafkaProp = buildAdminClientConfig(props, propPrefix); for (int currentAttempt = 1; currentAttempt <= maxAttempts && ret == null; currentAttempt++) { try (AdminClient admin = AdminClient.create(kafkaProp)) { @@ -92,7 +77,7 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP LOG.info("Topic '{}' configs: {}", topicName, topicConfigs); } - admin.createTopics(Collections.singletonList(topic)).all().get(); + createTopicIgnoringAlreadyExists(admin, topic); ret = topic.name(); @@ -143,6 +128,116 @@ public static String createAuditsTopicIfNotExists(Properties props, String propP return ret; } + /** + * Create the compacted partition-plan registry topic if missing. + * Always exactly one partition ({@link AuditServerConstants#PARTITION_PLAN_TOPIC_PARTITION_COUNT}); + * partition count is not configurable. A single partition is required so every plan version compacts + * under one key on one log end. Safe when multiple ingestor pods start together (concurrent topic creation). + */ + public static String createPartitionPlanTopicIfNotExists(Properties props, String propPrefix) { + String planTopic = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + short replicationFactor = (short) MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REPLICATION_FACTOR, AuditServerConstants.DEFAULT_REPLICATION_FACTOR); + int planTopicPartitions = AuditServerConstants.PARTITION_PLAN_TOPIC_PARTITION_COUNT; + int maxAttempts = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_MAX_RETRIES, 10) + 1; + int retryDelayMs = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_KAFKA_TOPIC_INIT_RETRY_DELAY_MS, 3000); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try (AdminClient admin = AdminClient.create(adminConfig)) { + Set topicNames = admin.listTopics().names().get(); + if (!topicNames.contains(planTopic)) { + LOG.info("Creating partition plan topic '{}' with {} partition and replication factor {}", planTopic, planTopicPartitions, replicationFactor); + NewTopic topic = new NewTopic(planTopic, planTopicPartitions, replicationFactor); + topic.configs(Collections.singletonMap("cleanup.policy", AuditServerConstants.KAFKA_TOPIC_CLEANUP_POLICY_COMPACT)); + createTopicIgnoringAlreadyExists(admin, topic); + AuditServerUtils.waitUntilTopicReady(admin, planTopic, Duration.ofSeconds(60)); + int partitionCount = describeTopicPartitionCount(admin, planTopic); + LOG.info("Partition plan topic '{}' is ready with {} partition(s)", planTopic, partitionCount); + } else { + int partitionCount = describeTopicPartitionCount(admin, planTopic); + requirePlanTopicPartitionCount(planTopic, partitionCount, planTopicPartitions); + LOG.info("Partition plan topic '{}' already exists with {} partition(s)", planTopic, partitionCount); + } + return planTopic; + } catch (Exception ex) { + if (attempt < maxAttempts) { + LOG.warn("Failed to ensure partition plan topic on attempt {}/{}. Retrying in {} ms. Error: {}", attempt, maxAttempts, retryDelayMs, ex.getMessage()); + try { + Thread.sleep(retryDelayMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } else { + LOG.error("Failed to create partition plan topic '{}' after {} attempts", planTopic, attempt, ex); + throw new RuntimeException("Failed to create partition plan topic '" + planTopic + "' after " + attempt + " attempts", ex); + } + } + } + throw new RuntimeException("Failed to create partition plan topic '" + planTopic + "'"); + } + + /** + * Returns whether the compacted partition-plan registry topic already exists on the audit Kafka cluster. + * When the check fails (broker unreachable, ACL denied), returns {@code false} so callers can fall back + * to static XML {@code auth_to_local} rules until the plan topic is available. + */ + public static boolean partitionPlanTopicExists(Properties props, String propPrefix) { + String planTopic = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + Set topicNames = admin.listTopics().names().get(); + boolean exists = topicNames.contains(planTopic); + LOG.debug("Partition plan topic '{}' exists: {}", planTopic, exists); + return exists; + } catch (Exception ex) { + LOG.warn("Could not determine whether partition plan topic '{}' exists: {}. Assuming it does not.", planTopic, ex.getMessage()); + return false; + } + } + + public static Map buildAdminClientConfig(Properties props, String propPrefix) { + String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + String saslMechanism = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM); + int connMaxIdleTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_CONN_MAX_IDEAL_MS, 10000); + int reqTimeoutMS = MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, 5000); + + Map kafkaProp = new HashMap<>(); + kafkaProp.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + kafkaProp.put("sasl.mechanism", saslMechanism); + kafkaProp.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + if (securityProtocol != null && securityProtocol.toUpperCase().contains("SASL")) { + kafkaProp.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, getJAASConfig(props, propPrefix)); + kafkaProp.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + } + kafkaProp.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, reqTimeoutMS); + kafkaProp.put(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, connMaxIdleTimeoutMS); + return kafkaProp; + } + + private static int describeTopicPartitionCount(AdminClient admin, String topicName) throws Exception { + TopicDescription topicDescription = admin.describeTopics(Collections.singletonList(topicName)).values().get(topicName).get(); + return topicDescription.partitions().size(); + } + + private static void requirePlanTopicPartitionCount(String planTopic, int actualPartitions, int expectedPartitions) { + if (actualPartitions != expectedPartitions) { + throw new RuntimeException("Partition plan topic '" + planTopic + "' must have exactly " + expectedPartitions + " partition(s) for compacted registry semantics, but has " + actualPartitions); + } + } + + private static void createTopicIgnoringAlreadyExists(AdminClient admin, NewTopic topic) throws Exception { + try { + admin.createTopics(Collections.singletonList(topic)).all().get(); + } catch (ExecutionException ex) { + if (!(ex.getCause() instanceof TopicExistsException)) { + throw ex; + } + LOG.info("Topic '{}' already exists", topic.name()); + } + } + public static String getJAASConfig(Properties props, String propPrefix) { // Use ranger service principal and keytab for Kafka authentication // This ensures consistent identity across all Ranger services and destination writes @@ -215,6 +310,22 @@ public static String getJAASConfig(Properties props, String propPrefix) { return jaasConfig; } + /** Grows the audit topic partition count when the plan requires more partitions than exist today. */ + public static void ensureTopicPartitionCount(Properties props, String propPrefix, String topicName, int requiredPartitions) { + LOG.info("Ensuring topic '{}' has at least {} partitions", topicName, requiredPartitions); + Map adminConfig = buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + updateExistingTopicPartitions(admin, topicName, requiredPartitions); + LOG.info("Topic '{}' partition count satisfied (required >= {})", topicName, requiredPartitions); + } catch (Exception e) { + LOG.error("Failed to ensure partition count for topic '{}' (required >= {})", topicName, requiredPartitions, e); + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException("Failed to ensure partition count for topic '" + topicName + "'", e); + } + } + private static String updateExistingTopicPartitions(AdminClient admin, String topicName, int partitions) { LOG.info("==> AuditMessageQueueUtils:updateExistingTopicPartitions() topic: {}, desired partitions: {}", topicName, partitions); diff --git a/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java index e20a6964743..950a7d477f6 100644 --- a/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java +++ b/audit-server/audit-common/src/test/java/org/apache/ranger/audit/utils/AuditMessageQueueUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.ranger.audit.utils; +import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.Test; @@ -65,4 +66,16 @@ public void testBuildTopicConfigsMapsAllSetProperties() { assertEquals("lz4", configs.get("compression.type")); assertEquals("2", configs.get("min.insync.replicas")); } + + @Test + public void testBuildAdminClientConfigUsesBootstrapServers() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS, "kafka:9092"); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, "PLAINTEXT"); + + Map adminConfig = AuditMessageQueueUtils.buildAdminClientConfig(props, PROP_PREFIX); + + assertEquals("kafka:9092", adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG)); + assertEquals("PLAINTEXT", adminConfig.get(AdminClientConfig.SECURITY_PROTOCOL_CONFIG)); + } } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java index 21684098963..800ebda4ea6 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditMessageQueue.java @@ -23,6 +23,8 @@ import org.apache.ranger.audit.destination.AuditDestination; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanWatcher; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.utils.AuditMessageQueueUtils; import org.slf4j.Logger; @@ -48,6 +50,7 @@ public class AuditMessageQueue extends AuditDestination { private String topicName; private Thread producerThread; private AuditRecoveryManager recoveryManager; + private PartitionPlanWatcher partitionPlanWatcher; @Override public void init(Properties props, String propPrefix) { @@ -56,6 +59,7 @@ public void init(Properties props, String propPrefix) { super.init(props, propPrefix); createAuditsTopic(props, PROP_INGESTOR_PREFIX); + startPartitionPlanWatcherIfEnabled(props); createKafkaProducer(props, PROP_INGESTOR_PREFIX); createRecoveryManager(props, PROP_INGESTOR_PREFIX); @@ -75,6 +79,16 @@ public void start() { public void stop() { LOG.info("==> AuditMessageQueue.stop() [CORE AUDIT SERVER]"); + if (partitionPlanWatcher != null) { + try { + partitionPlanWatcher.stop(); + } catch (Exception e) { + LOG.error("Error stopping partition plan watcher", e); + } finally { + partitionPlanWatcher = null; + } + } + // Shutdown recovery manager first to process any remaining messages if (recoveryManager != null) { try { @@ -340,6 +354,22 @@ private void createAuditsTopic(final Properties props, final String propPrefix) } } + private void startPartitionPlanWatcherIfEnabled(final Properties props) { + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(props, PROP_INGESTOR_PREFIX)) { + return; + } + if (topicName == null) { + throw new IllegalStateException("Audit topic must be created before starting partition plan watcher"); + } + try { + partitionPlanWatcher = new PartitionPlanWatcher(props, PROP_INGESTOR_PREFIX, topicName, null); + partitionPlanWatcher.startBlocking(); + } catch (Exception e) { + LOG.error("Failed to start partition plan watcher for audit topic '{}'", topicName, e); + throw new RuntimeException("Failed to start partition plan watcher for dynamic partitioning", e); + } + } + private void createRecoveryManager(final Properties props, final String propPrefix) { // Create recovery manager even if kafkaProducer is null - it will handle null producer gracefully // This ensures audits are spooled when Kafka is unavailable during startup diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java index d0832bdaa60..f9a0c837368 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java @@ -22,6 +22,10 @@ import org.apache.kafka.clients.producer.Partitioner; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.PartitionInfo; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; import org.apache.ranger.audit.server.AuditServerConstants; import org.apache.ranger.audit.utils.AuditServerLogFormatter; import org.slf4j.Logger; @@ -50,11 +54,19 @@ public class AuditPartitioner implements Partitioner { private int[] configuredPluginPartitionEnd; private int bufferPartitionStart; private int bufferPartitionCount; + private boolean dynamicPlanEnabled; private final ConcurrentHashMap appIdCounters = new ConcurrentHashMap<>(); @Override public void configure(Map configs) { String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + String ingestorPropPrefix = propPrefix.substring(0, propPrefix.length() - 1); + dynamicPlanEnabled = PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configs, ingestorPropPrefix); + if (dynamicPlanEnabled) { + LOG.info("Dynamic partition plan enabled — routing from in-memory plan (PartitionPlanHolder)"); + logDynamicPlanConfiguration(configs, propPrefix); + return; + } String pluginsStr = getConfig(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); configuredPlugins = pluginsStr.split(","); @@ -116,13 +128,21 @@ public void configure(Map configs) { @Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { + String appId = key != null ? key.toString() : null; + if (dynamicPlanEnabled) { + int numPartitions = resolveTopicPartitionCount(cluster, topic); + if (appId == null || appId.isEmpty()) { + return Math.abs(System.identityHashCode(key) % numPartitions); + } + return partitionFromPlan(appId, numPartitions); + } + int numPartitions = totalPartitions; List partitions = cluster.partitionsForTopic(topic); if (partitions != null && !partitions.isEmpty()) { numPartitions = partitions.size(); } - String appId = key != null ? key.toString() : null; if (appId == null || appId.isEmpty()) { return Math.abs(System.identityHashCode(key) % numPartitions); } @@ -135,10 +155,8 @@ public int partition(String topic, Object key, byte[] keyBytes, Object value, by end = start; } int rangeSize = end - start + 1; - int subPartition = appIdCounters - .computeIfAbsent(appId, k -> new AtomicInteger(0)) - .getAndIncrement() % rangeSize; - return start + subPartition; + int roundRobinIndex = nextRoundRobinIndex(appId, rangeSize); + return start + roundRobinIndex; } else { // Unconfigured plugin - use buffer partitions int start = Math.min(bufferPartitionStart, numPartitions - 1); @@ -156,6 +174,164 @@ public void close() { appIdCounters.clear(); } + /** + * Routes one audit event to a Kafka partition using the in-memory dynamic partition plan. + * + *

The plan splits the audit topic into dedicated plugin lanes (configured plugins + * such as {@code hdfs}, {@code hiveServer}) and a shared buffer pool (everything else). + * Routing for a given {@code appId} follows this order: + *

    + *
  1. Known plugin with dedicated partitions — round-robin across that plugin's + * assignment list so load is spread evenly while preserving per-plugin ordering lanes. + * Example: {@code hdfs} → [0, 1, 2] sends three successive events to 0, then 1, + * then 2, then wraps.
    2. + *
  2. Unknown or unconfigured plugin — sticky hash into the shared buffer pool so + * the same {@code appId} always lands on the same buffer partition. + * Example: buffer → [10, 11]; {@code myCustomApp} consistently maps to 10 or 11.
    3. + *
  3. No buffer partitions in the plan — sticky hash across the full topic when every + * partition is assigned to configured plugins.
    4. + *
+ * + *

After topic scale-up, the Kafka producer's cluster metadata can lag behind + * {@link PartitionPlan#getTopicPartitionCount()}. We use the larger of the two counts as + * {@code effectiveTopicPartitionCount} so a newly planned tail id (e.g. 12) is not folded + * into the stale metadata ceiling (e.g. 11). + * + * @param appId plugin key from the audit event (Kafka record key) + * @param kafkaClusterPartitionCount partition count reported by live Kafka cluster metadata + * @return Kafka partition id to produce to + */ + private int partitionFromPlan(String appId, int kafkaClusterPartitionCount) { + PartitionPlan plan = PartitionPlanHolder.getInstance().getPlan(); + if (plan == null) { + LOG.error("Dynamic partition plan is not loaded; falling back to hash routing for appId '{}'", appId); + return hashAppIdToPartitionIndex(appId, kafkaClusterPartitionCount); + } + + int effectiveTopicPartitionCount = Math.max(kafkaClusterPartitionCount, plan.getTopicPartitionCount()); + + PluginPartitionAssignment pluginAssignment = findPluginAssignment(plan, appId); + if (pluginAssignment != null && !pluginAssignment.getPartitions().isEmpty()) { + List dedicatedPluginPartitions = pluginAssignment.getPartitions(); + int dedicatedLaneIndex = nextRoundRobinIndex(appId, dedicatedPluginPartitions.size()); + int plannedPartitionId = dedicatedPluginPartitions.get(dedicatedLaneIndex); + return resolvePlannedPartitionId(plannedPartitionId, effectiveTopicPartitionCount); + } + + List sharedBufferPartitions = plan.getBuffer().getPartitions(); + if (sharedBufferPartitions.isEmpty()) { + return hashAppIdToPartitionIndex(appId, effectiveTopicPartitionCount); + } + int bufferPoolIndex = hashAppIdToPartitionIndex(appId, sharedBufferPartitions.size()); + int plannedPartitionId = sharedBufferPartitions.get(bufferPoolIndex); + return resolvePlannedPartitionId(plannedPartitionId, effectiveTopicPartitionCount); + } + + /** + * Sticky hash: same {@code appId} always picks the same slot in {@code [0, slotCount)}. + * Used for buffer-pool routing and for plan-not-loaded fallback. + */ + private static int hashAppIdToPartitionIndex(String appId, int slotCount) { + return Math.abs(appId.hashCode() % slotCount); + } + + /** + * Returns the next dedicated-lane index for round-robin routing within one plugin's partition set. + * Each {@code appId} keeps its own counter (0, 1, 2, …); {@code % dedicatedLaneCount} cycles + * through that plugin's lanes only. + */ + private int nextRoundRobinIndex(String appId, int dedicatedLaneCount) { + AtomicInteger messageCounter = appIdCounters.computeIfAbsent(appId, k -> new AtomicInteger(0)); + return messageCounter.getAndIncrement() % dedicatedLaneCount; + } + + /** Looks up a plugin assignment using case-insensitive plugin id matching. */ + private static PluginPartitionAssignment findPluginAssignment(PartitionPlan plan, String appId) { + for (Map.Entry entry : plan.getPlugins().entrySet()) { + if (entry.getKey().equalsIgnoreCase(appId)) { + return entry.getValue(); + } + } + return null; + } + + /** Returns the live partition count for the audit topic from the Kafka cluster metadata. */ + private static int resolveTopicPartitionCount(Cluster cluster, String topic) { + List partitions = cluster.partitionsForTopic(topic); + if (partitions != null && !partitions.isEmpty()) { + return partitions.size(); + } + return 1; + } + + /** + * Converts a partition id from the plan into the id passed to the Kafka producer. + * + *

When cluster metadata is stale after scale-up, {@code effectiveTopicPartitionCount} may + * exceed what the broker metadata reports. Planned tail ids must still be returned as-is — + * never clamp partition 12 down to 11 just because metadata has not caught up yet. + * + * @param plannedPartitionId partition id from the dynamic plan assignment or buffer pool + * @param effectiveTopicPartitionCount {@code max(kafkaClusterPartitionCount, plan.topicPartitionCount)} + */ + private static int resolvePlannedPartitionId(int plannedPartitionId, int effectiveTopicPartitionCount) { + if (effectiveTopicPartitionCount <= 0) { + return 0; + } + if (plannedPartitionId < 0) { + return 0; + } + if (plannedPartitionId < effectiveTopicPartitionCount) { + return plannedPartitionId; + } + return plannedPartitionId; + } + + /** Logs the dynamic plan snapshot installed in {@link PartitionPlanHolder} at startup. */ + private void logDynamicPlanConfiguration(Map configs, String propPrefix) { + int defaultPerPlugin = getIntConfig(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + if (defaultPerPlugin < 1) { + defaultPerPlugin = 1; + } + + PartitionPlan plan = PartitionPlanHolder.getInstance().getPlan(); + AuditServerLogFormatter.LogBuilder logBuilder = AuditServerLogFormatter.builder("****** AuditPartitioner Configuration ******"); + if (plan == null) { + logBuilder.add("Mode: ", "dynamic (plan not loaded yet)"); + logBuilder.logInfo(LOG); + return; + } + + logBuilder.add("Mode: ", "dynamic (PartitionPlanHolder)"); + logBuilder.add("Plan version: ", plan.getVersion()); + logBuilder.add("Total partitions: ", plan.getTopicPartitionCount()); + logBuilder.add("Configured plugins: ", plan.getPlugins().size()); + logBuilder.add("Default partitions per plugin: ", defaultPerPlugin); + for (Map.Entry entry : plan.getPlugins().entrySet()) { + List partitionIds = entry.getValue().getPartitions(); + String rangeInfo = formatPartitionRangeInfo(partitionIds); + logBuilder.add("Plugin '" + entry.getKey() + "'", rangeInfo); + } + logBuilder.add("Buffer partitions (unconfigured): ", formatBufferPartitionInfo(plan.getBuffer().getPartitions())); + logBuilder.logInfo(LOG); + } + + private static String formatPartitionRangeInfo(List partitionIds) { + if (partitionIds.isEmpty()) { + return "no partitions assigned in plan"; + } + return String.format("%d partitions (range: %d-%d)", + partitionIds.size(), partitionIds.get(0), partitionIds.get(partitionIds.size() - 1)); + } + + private static String formatBufferPartitionInfo(List bufferPartitionIds) { + if (bufferPartitionIds.isEmpty()) { + return "none (all topic partitions assigned to configured plugins)"; + } + return String.format("%d partitions (range: %d-%d)", + bufferPartitionIds.size(), bufferPartitionIds.get(0), bufferPartitionIds.get(bufferPartitionIds.size() - 1)); + } + private int indexOfConfiguredPlugin(String appId) { for (int i = 0; i < configuredPlugins.length; i++) { if (configuredPlugins[i].equalsIgnoreCase(appId)) { diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java new file mode 100644 index 00000000000..a4314b7e156 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** Parses and composes Kerberos {@code auth_to_local} rules from {@code ranger.audit.ingestor.auth.to.local}. */ +public class AuthToLocalRuleCatalog { + private static final Pattern RULE_SUBSTITUTION_SHORT_NAME_PATTERN = + Pattern.compile("s/\\.\\*/([^/]+)/\\s*$"); + private static final String GENERATED_SHORT_NAME_RULE_TEMPLATE = + "RULE:[2:$1/$2@$0](%s/.*@.*)s/.*/%s/"; + + private final List primaryCatalogRulesInOrder; + private final List fallbackRuleLines; + + AuthToLocalRuleCatalog(List primaryCatalogRulesInOrder, List fallbackRuleLines) { + this.primaryCatalogRulesInOrder = List.copyOf(primaryCatalogRulesInOrder); + this.fallbackRuleLines = List.copyOf(fallbackRuleLines); + } + + public static AuthToLocalRuleCatalog parse(String rawAuthToLocalRules) { + List primaryCatalogRulesInOrder = new ArrayList<>(); + List fallbackRuleLines = new ArrayList<>(); + for (String ruleLine : tokenizeRuleLines(rawAuthToLocalRules)) { + if (isFallbackKerberosRule(ruleLine)) { + fallbackRuleLines.add(ruleLine); + } else { + primaryCatalogRulesInOrder.add(new PrimaryCatalogRule(ruleLine, extractMappedShortName(ruleLine))); + } + } + return new AuthToLocalRuleCatalog(primaryCatalogRulesInOrder, fallbackRuleLines); + } + + /** Full static rule set (all primary catalog rules in order + fallback tail). */ + public String composeFullCatalogRules() { + List composedRuleLines = new ArrayList<>(primaryCatalogRulesInOrder.size() + fallbackRuleLines.size()); + for (PrimaryCatalogRule catalogRule : primaryCatalogRulesInOrder) { + composedRuleLines.add(catalogRule.ruleLine); + } + composedRuleLines.addAll(fallbackRuleLines); + return joinRuleLines(composedRuleLines); + } + + /** + * Builds the active rule set for dynamic mode: catalog rules whose mapped short name is in the union, + * plus generated rules for union members not covered by the catalog, plus fallback tail. + * + *

Examples (union member to effective rule): + *

    + *
  • {@code hdfs} in union: full hdfs catalog line (nn/dn/jn/hdfs principals map to hdfs)
    • + *
  • {@code nn} in union but catalog mapped name is hdfs: generated nn rule plus hdfs catalog line
    • + *
  • {@code foo} in union, not in catalog: generated foo/host@REALM to foo rule
    • + *
+ * Empty union falls back to {@link #composeFullCatalogRules()}. + */ + public String composeRulesForAllowedShortNames(Set allowedUserShortNames) { + if (allowedUserShortNames == null || allowedUserShortNames.isEmpty()) { + return composeFullCatalogRules(); + } + + Set normalizedAllowedShortNames = allowedUserShortNames.stream() + .filter(StringUtils::isNotBlank) + .map(String::trim) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (normalizedAllowedShortNames.isEmpty()) { + return composeFullCatalogRules(); + } + + List composedRuleLines = new ArrayList<>(); + Set shortNamesMatchedByCatalogRule = new LinkedHashSet<>(); + + for (PrimaryCatalogRule catalogRule : primaryCatalogRulesInOrder) { + if (catalogRule.mappedShortName != null && normalizedAllowedShortNames.contains(catalogRule.mappedShortName)) { + composedRuleLines.add(catalogRule.ruleLine); + shortNamesMatchedByCatalogRule.add(catalogRule.mappedShortName); + } + } + + List generatedSimpleRuleLines = normalizedAllowedShortNames.stream() + .filter(shortName -> !shortNamesMatchedByCatalogRule.contains(shortName)) + .sorted() + .map(AuthToLocalRuleCatalog::buildGeneratedShortNameRule) + .collect(Collectors.toList()); + composedRuleLines.addAll(generatedSimpleRuleLines); + composedRuleLines.addAll(fallbackRuleLines); + return joinRuleLines(composedRuleLines); + } + + public int getPrimaryCatalogRuleCount() { + return primaryCatalogRulesInOrder.size(); + } + + /** + * Mapped Kerberos short name from a catalog rule line (substitution suffix s/.star/<name>/). + * Example: hdfs catalog rule maps nn/dn/jn/hdfs principals to short name hdfs. + */ + public static String extractMappedShortName(String ruleLine) { + if (StringUtils.isBlank(ruleLine)) { + return null; + } + Matcher substitutionMatcher = RULE_SUBSTITUTION_SHORT_NAME_PATTERN.matcher(ruleLine); + if (substitutionMatcher.find()) { + return substitutionMatcher.group(1); + } + return null; + } + + private static List tokenizeRuleLines(String rawAuthToLocalRules) { + if (StringUtils.isBlank(rawAuthToLocalRules)) { + return Collections.emptyList(); + } + return Arrays.stream(rawAuthToLocalRules.split("\\s+")) + .map(String::trim) + .filter(ruleLine -> ruleLine.startsWith("RULE:") || "DEFAULT".equals(ruleLine)) + .collect(Collectors.toList()); + } + + private static boolean isFallbackKerberosRule(String ruleLine) { + return "DEFAULT".equals(ruleLine) || ruleLine.startsWith("RULE:[1:") || ruleLine.contains("s/@.*//"); + } + + /** Simple rule for allowlist short names absent from the XML catalog: {@code myapp/host@REALM -> myapp}. */ + private static String buildGeneratedShortNameRule(String shortName) { + return String.format(GENERATED_SHORT_NAME_RULE_TEMPLATE, shortName, shortName); + } + + private static String joinRuleLines(List ruleLines) { + return String.join("\n", ruleLines); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java new file mode 100644 index 00000000000..95184a6f296 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * Builds effective {@code auth_to_local} rules from the XML catalog and the global allowlist union + * ({@link #collectAllowedUserShortNames}). Rule text is not stored in the partition-plan Kafka JSON. + * + *

Two-step audit POST identity (see {@link ServiceAllowlistResolver}): + *

    + *
  1. Kerberos mapping — {@code KerberosName.getShortName()} uses composed rules (global union)
    2. + *
  2. Authorization — per-repo {@code services[repo].allowedUsers} (or static XML fallback)
    3. + *
+ */ +public class AuthToLocalRuleComposer { + private static final Logger LOG = LoggerFactory.getLogger(AuthToLocalRuleComposer.class); + + private static final AuthToLocalRuleComposer INSTANCE = new AuthToLocalRuleComposer(); + + private volatile AuthToLocalRuleCatalog ruleCatalog; + private volatile String lastAppliedRuleText; + private volatile int lastAppliedPartitionPlanVersion; + private volatile Boolean partitionPlanTopicExistsTestOverride; + + private AuthToLocalRuleComposer() { + } + + public static AuthToLocalRuleComposer getInstance() { + return INSTANCE; + } + + /** Loads the rule catalog from {@code ranger.audit.ingestor.auth.to.local} site XML. */ + public synchronized void initializeFromConfig() { + Properties ingestorProperties = AuditServerConfig.getInstance().getProperties(); + initializeFromProperties(ingestorProperties); + } + + public synchronized void initializeFromProperties(Properties ingestorProperties) { + String rawAuthToLocalRules = ingestorProperties.getProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL); + ruleCatalog = AuthToLocalRuleCatalog.parse(rawAuthToLocalRules); + lastAppliedRuleText = null; + lastAppliedPartitionPlanVersion = 0; + LOG.debug("Loaded auth_to_local catalog with {} primary rules", ruleCatalog.getPrimaryCatalogRuleCount()); + } + + /** Applies the full XML catalog (non-dynamic / startup fallback). */ + public void applyStaticRules() { + AuthToLocalRuleCatalog loadedRuleCatalog = requireRuleCatalog(); + applyKerberosNameRules(loadedRuleCatalog.composeFullCatalogRules(), 0); + } + + /** + * Dynamic-mode startup: when the partition-plan Kafka topic does not exist yet, apply the full XML + * catalog so Kerberos mapping works before {@link PartitionPlanWatcher} bootstraps the registry. + * When the topic already exists, defer to composed rules on {@link PartitionPlanHolder#install}. + */ + public void applyStartupRulesForDynamicMode(Properties ingestorProperties, String ingestorPropertyPrefix) { + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(ingestorProperties, ingestorPropertyPrefix)) { + return; + } + requireRuleCatalog(); + if (isPartitionPlanTopicPresent(ingestorProperties, ingestorPropertyPrefix)) { + LOG.info("Partition plan topic exists; auth_to_local rules will be composed from allowlisted short names on plan install"); + } else { + applyStaticRules(); + LOG.info("Partition plan topic does not exist yet; applied full auth_to_local catalog from XML until plan bootstrap"); + } + } + + /** + * When dynamic partition-plan mode is enabled, compose {@code auth_to_local} rules from the XML + * catalog plus the global allowlist union, then install via {@link KerberosName#setRules(String)}. + * + *

Called from {@link PartitionPlanHolder#install} on every plan version. Example plan: + * 

{@code
+     * services: {
+     *   dev_hdfs:  { allowedUsers: [hdfs, nn] },
+     *   dev_hive:  { allowedUsers: [hive] },
+     *   dev_foo:   { allowedUsers: [foo] }   // new plugin, not in XML catalog
+     * }
+     * }
+ * Union of hdfs, nn, hive, foo activates hdfs catalog rule (nn/dn/jn/hdfs principals to hdfs), + * hive catalog rule, and a generated foo/host@REALM to foo rule. Per-repo POST checks still + * use each repo's own allowlist, not the union. + */ + public void applyForPlan(PartitionPlan partitionPlan) { + Properties ingestorProperties = AuditServerConfig.getInstance().getProperties(); + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(ingestorProperties, PartitionPlanService.INGESTOR_PROP_PREFIX)) { + return; + } + if (partitionPlan == null) { + return; + } + + AuthToLocalRuleCatalog loadedRuleCatalog = requireRuleCatalog(); + Set allowedUserShortNames = collectAllowedUserShortNames(partitionPlan); + String composedRuleText = loadedRuleCatalog.composeRulesForAllowedShortNames(allowedUserShortNames); + applyKerberosNameRules(composedRuleText, partitionPlan.getVersion()); + LOG.info("Applied composed auth_to_local rules for plan version {} ({} active short names)", + partitionPlan.getVersion(), allowedUserShortNames.size()); + } + + /** Visible for tests — composes without applying Kerberos rules. */ + public String composeKerberosRulesForAllowedShortNames(Set allowedUserShortNames) { + return requireRuleCatalog().composeRulesForAllowedShortNames(allowedUserShortNames); + } + + /** Clears cached apply state between unit tests. */ + public synchronized void resetForTests() { + lastAppliedRuleText = null; + lastAppliedPartitionPlanVersion = 0; + partitionPlanTopicExistsTestOverride = null; + } + + /** When non-null, overrides {@link AuditMessageQueueUtils#partitionPlanTopicExists} for unit tests. */ + public void setPartitionPlanTopicExistsTestOverride(Boolean topicExists) { + partitionPlanTopicExistsTestOverride = topicExists; + } + + private boolean isPartitionPlanTopicPresent(Properties ingestorProperties, String ingestorPropertyPrefix) { + Boolean topicExistsOverride = partitionPlanTopicExistsTestOverride; + if (topicExistsOverride != null) { + return topicExistsOverride; + } + return AuditMessageQueueUtils.partitionPlanTopicExists(ingestorProperties, ingestorPropertyPrefix); + } + + /** + * Global allowlist union: all distinct {@code allowedUsers} short names across {@code plan.services}. + * Used only for {@code auth_to_local} composition (which mapping rules are active), not for POST authorization. + * + *

Example: dev_hdfs with hdfs and nn, dev_hive with hive, dev_ozone with om and ozone yields + * union hdfs, nn, hive, om, ozone. Repos not listed in services do not contribute. + */ + public static Set collectAllowedUserShortNames(PartitionPlan partitionPlan) { + Set allowedUserShortNames = new LinkedHashSet<>(); + if (partitionPlan == null || partitionPlan.getServices() == null) { + return allowedUserShortNames; + } + for (Map.Entry serviceEntry : partitionPlan.getServices().entrySet()) { + ServiceAllowlistEntry allowlistEntry = serviceEntry.getValue(); + if (allowlistEntry == null) { + continue; + } + for (String allowedUserShortName : allowlistEntry.getAllowedUsers()) { + if (StringUtils.isNotBlank(allowedUserShortName)) { + allowedUserShortNames.add(allowedUserShortName.trim()); + } + } + } + return allowedUserShortNames; + } + + private AuthToLocalRuleCatalog requireRuleCatalog() { + AuthToLocalRuleCatalog loadedRuleCatalog = ruleCatalog; + if (loadedRuleCatalog == null) { + initializeFromConfig(); + loadedRuleCatalog = ruleCatalog; + } + if (loadedRuleCatalog == null) { + throw new IllegalStateException("auth_to_local catalog is not loaded"); + } + return loadedRuleCatalog; + } + + private synchronized void applyKerberosNameRules(String composedRuleText, int partitionPlanVersion) { + if (StringUtils.isBlank(composedRuleText)) { + LOG.warn("Skipping auth_to_local apply: composed rules are blank"); + return; + } + if (composedRuleText.equals(lastAppliedRuleText) && partitionPlanVersion == lastAppliedPartitionPlanVersion) { + return; + } + try { + KerberosName.setRules(composedRuleText); + lastAppliedRuleText = composedRuleText; + lastAppliedPartitionPlanVersion = partitionPlanVersion; + LOG.debug("KerberosName auth_to_local rules updated (partitionPlanVersion={})", partitionPlanVersion); + } catch (Exception e) { + LOG.error("Failed to apply composed auth_to_local rules for plan version {}: {}", + partitionPlanVersion, e.getMessage(), e); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java new file mode 100644 index 00000000000..029acc813c2 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaAuditTopicPartitionGrower.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; + +import java.util.Properties; + +/** Grows the audit topic partition count before a plan references new tail IDs. */ +public class KafkaAuditTopicPartitionGrower { + /** Increases {@code ranger_audits} partition count when the plan needs more lanes. */ + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + AuditMessageQueueUtils.ensureTopicPartitionCount(props, propPrefix, auditTopicName, requiredPartitionCount); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java new file mode 100644 index 00000000000..198219ae5d2 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/KafkaPartitionPlanRegistry.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; + +/** Kafka compacted topic implementation of {@link PartitionPlanRegistry}. */ +public class KafkaPartitionPlanRegistry implements PartitionPlanRegistry { + private static final Logger LOG = LoggerFactory.getLogger(KafkaPartitionPlanRegistry.class); + + private final Properties props; + private final String propPrefix; + private final String planTopic; + private final int consumerPollTimeoutMs; + private final KafkaProducer producer; + + public KafkaPartitionPlanRegistry(Properties props, String propPrefix) throws Exception { + this.props = props; + this.propPrefix = propPrefix; + this.planTopic = PartitionPlanKafkaConfig.resolvePlanTopicName(props, propPrefix); + this.consumerPollTimeoutMs = PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, propPrefix); + AuditMessageQueueUtils.createPartitionPlanTopicIfNotExists(props, propPrefix); + this.producer = new KafkaProducer<>(PartitionPlanKafkaConfig.producerConfig(props, propPrefix)); + LOG.info("Kafka partition plan registry ready for topic '{}'", planTopic); + } + + /** Reads the latest compacted value for the audit topic key from partition 0. */ + @Override + public PartitionPlan readPlan(String auditTopicKey) { + requireAuditTopicKey(auditTopicKey); + try (KafkaConsumer consumer = openConsumer()) { + return readLatestCompactedPlan(consumer, auditTopicKey); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + LOG.error("Failed to read partition plan from Kafka topic '{}' for audit topic key '{}'", planTopic, auditTopicKey, e); + throw new PartitionPlanException("Failed to read partition plan from Kafka topic '" + planTopic + "'", e); + } + } + + /** Publishes a new plan version to the compacted topic (key = audit topic name). */ + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + requireAuditTopicKey(auditTopicKey); + if (plan == null) { + throw new PartitionPlanException("Partition plan is required"); + } + PartitionPlanValidator.validate(plan); + try { + producer.send(new ProducerRecord<>(planTopic, auditTopicKey, plan.toJson())).get(); + LOG.info("Wrote partition plan version {} for audit topic '{}' to '{}'", plan.getVersion(), auditTopicKey, planTopic); + } catch (Exception e) { + LOG.error("Failed to write partition plan version {} to Kafka topic '{}' for audit topic key '{}'", plan.getVersion(), planTopic, auditTopicKey, e); + throw new PartitionPlanException("Failed to write partition plan to Kafka topic '" + planTopic + "'", e); + } + } + + @Override + public void close() { + producer.flush(); + producer.close(); + } + + private KafkaConsumer openConsumer() throws Exception { + return new KafkaConsumer<>(PartitionPlanKafkaConfig.consumerConfig(props, propPrefix, PartitionPlanConstants.PLAN_REGISTRY_CONSUMER_GROUP)); + } + + private PartitionPlan readLatestCompactedPlan(KafkaConsumer consumer, String auditTopicKey) { + TopicPartition partition = new TopicPartition(planTopic, 0); + consumer.assign(Collections.singletonList(partition)); + consumer.seekToBeginning(Collections.singletonList(partition)); + + PartitionPlan latest = null; + ConsumerRecords records; + do { + records = consumer.poll(Duration.ofMillis(consumerPollTimeoutMs)); + for (ConsumerRecord record : records) { + if (auditTopicKey.equals(record.key())) { + latest = PartitionPlan.fromJson(record.value()); + } + } + } while (!records.isEmpty()); + return latest; + } + + private static void requireAuditTopicKey(String auditTopicKey) { + if (StringUtils.isBlank(auditTopicKey)) { + throw new PartitionPlanException("auditTopicKey is required"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java new file mode 100644 index 00000000000..a9456f19353 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** Append-only plan updates: promote unknown plugins and scale hot plugins without reshuffling. */ +public class PartitionPlanAllocator { + private PartitionPlanAllocator() { + } + + public static PartitionPlan promotePlugin(PartitionPlan current, String pluginId, int partitionCount, String updatedBy) { + return promotePlugin(current, pluginId, partitionCount, updatedBy, null, null); + } + + /** + * Give a plugin its own partitions. Uses buffer IDs first; adds new tail IDs when buffer is too small. + * Optionally upserts {@code services[repo]} in the same plan version when {@code repo} and {@code allowedUsers} are set. + */ + public static PartitionPlan promotePlugin(PartitionPlan current, String pluginId, int partitionCount, String updatedBy, String repo, List allowedUsers) { + requireMutationInputs(current, pluginId, partitionCount, updatedBy); + if (current.getPlugins().containsKey(pluginId)) { + assertPromoteNotConflicting(current, pluginId, partitionCount, repo, allowedUsers); + throw new PartitionPlanException("Plugin '" + pluginId + "' already has dedicated partitions"); + } + + List remainingBuffer = new ArrayList<>(current.getBuffer().getPartitions()); + List newPluginIds = takeFromBuffer(remainingBuffer, partitionCount); + int topicPartitionCount = appendTailPartitions(newPluginIds, current.getTopicPartitionCount(), partitionCount - newPluginIds.size()); + + Map plugins = addPluginAssignment(current, pluginId, newPluginIds); + Map services = mergeServiceAllowlist(current.getServices(), repo, allowedUsers); + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, remainingBuffer, services); + } + + /** + * Onboard a Ranger service repo: promote plugin partitions and upsert service allowlist atomically. + */ + public static PartitionPlan onboardRepo(PartitionPlan current, String repo, String pluginId, int partitionCount, List allowedUsers, String updatedBy) { + if (StringUtils.isBlank(repo)) { + throw new PartitionPlanException("repo is required"); + } + if (allowedUsers == null || allowedUsers.isEmpty()) { + throw new PartitionPlanException("allowedUsers are required"); + } + return promotePlugin(current, pluginId, partitionCount, updatedBy, repo, allowedUsers); + } + + /** + * Add more partitions to an existing plugin by appending new tail IDs only. + */ + public static PartitionPlan scalePlugin(PartitionPlan current, String pluginId, int additionalPartitions, String updatedBy) { + requireMutationInputs(current, pluginId, additionalPartitions, updatedBy); + if (!current.getPlugins().containsKey(pluginId)) { + throw new PartitionPlanException("Plugin '" + pluginId + "' is not configured; promote it first"); + } + + List pluginIds = new ArrayList<>(current.getPlugins().get(pluginId).getPartitions()); + int topicPartitionCount = appendTailPartitions(pluginIds, current.getTopicPartitionCount(), additionalPartitions); + + return commitPlanUpdate(current, updatedBy, topicPartitionCount, addPluginAssignment(current, pluginId, pluginIds), current.getBuffer().getPartitions(), current.getServices()); + } + + /** + * True when the plugin is already promoted with {@code partitionCount} slots and, when {@code repo} is set, + * the service allowlist already matches {@code allowedUsers}. + */ + public static boolean isPromoteAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { + if (current == null || !pluginHasPartitionCount(current, pluginId, partitionCount)) { + return false; + } + if (StringUtils.isNotBlank(repo)) { + ServiceAllowlistEntry existing = current.getServices().get(repo.trim()); + return existing != null && existing.hasSameAllowedUsers(allowedUsers); + } + return true; + } + + /** True when onboard request matches current allowlist and plugin assignment. */ + public static boolean isOnboardAlreadyApplied(PartitionPlan current, String repo, String pluginId, int partitionCount, List allowedUsers) { + return isPromoteAlreadyApplied(current, pluginId, partitionCount, repo, allowedUsers); + } + + /** Applies a merged plan with append-only checks against the current plan. */ + public static PartitionPlan replacePlan(PartitionPlan current, PartitionPlan proposed) { + if (current == null || proposed == null) { + throw new PartitionPlanException("Current and proposed plans are required"); + } + if (!StringUtils.equals(current.getTopic(), proposed.getTopic())) { + throw new PartitionPlanException("Proposed topic must match current topic"); + } + PartitionPlan next = proposed.toBuilder().version(current.getVersion() + 1).build(); + PartitionPlanValidator.validate(next); + PartitionPlanValidator.validateAppendOnly(current, next); + return next; + } + + /** Pull up to count partition IDs from the front of the buffer list. */ + private static List takeFromBuffer(List bufferIds, int count) { + List taken = new ArrayList<>(Math.min(count, bufferIds.size())); + while (taken.size() < count && !bufferIds.isEmpty()) { + taken.add(bufferIds.remove(0)); + } + return taken; + } + + /** Append new tail partition IDs and return the new topic partition count. */ + private static int appendTailPartitions(List target, int topicPartitionCount, int count) { + for (int i = 0; i < count; i++) { + target.add(topicPartitionCount++); + } + return topicPartitionCount; + } + + private static Map addPluginAssignment(PartitionPlan current, String pluginId, List partitionIds) { + Map plugins = new LinkedHashMap<>(current.getPlugins()); + plugins.put(pluginId, new PluginPartitionAssignment(partitionIds)); + return plugins; + } + + private static PartitionPlan commitPlanUpdate(PartitionPlan current, String updatedBy, int topicPartitionCount, Map plugins, List bufferIds, Map services) { + PartitionPlan next = current.toBuilder() + .version(current.getVersion() + 1) + .topicPartitionCount(topicPartitionCount) + .plugins(plugins) + .buffer(new PluginPartitionAssignment(bufferIds)) + .services(services != null ? services : current.getServices()) + .updatedAt(Instant.now().toString()) + .updatedBy(updatedBy) + .build(); + PartitionPlanValidator.validate(next); + PartitionPlanValidator.validateAppendOnly(current, next); + return next; + } + + private static Map mergeServiceAllowlist(Map currentServices, String repo, List allowedUsers) { + Map services = new LinkedHashMap<>(currentServices); + if (StringUtils.isNotBlank(repo) && allowedUsers != null && !allowedUsers.isEmpty()) { + services.put(repo.trim(), ServiceAllowlistEntry.ofUsers(allowedUsers)); + } + return services; + } + + private static boolean pluginHasPartitionCount(PartitionPlan current, String pluginId, int partitionCount) { + PluginPartitionAssignment assignment = current.getPlugins().get(pluginId); + return assignment != null && assignment.getPartitions().size() == partitionCount; + } + + private static void assertPromoteNotConflicting(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { + PluginPartitionAssignment existing = Objects.requireNonNull(current.getPlugins().get(pluginId)); + if (existing.getPartitions().size() != partitionCount) { + throw new PartitionPlanException("Plugin '" + pluginId + "' already has " + existing.getPartitions().size() + " dedicated partition(s); requested " + partitionCount); + } + if (StringUtils.isNotBlank(repo)) { + ServiceAllowlistEntry serviceEntry = current.getServices().get(repo.trim()); + if (serviceEntry != null && !serviceEntry.hasSameAllowedUsers(allowedUsers)) { + throw new PartitionPlanException("Service '" + repo.trim() + "' already exists with different allowedUsers"); + } + } + } + + private static void requireMutationInputs(PartitionPlan current, String pluginId, int partitionCount, String updatedBy) { + if (current == null) { + throw new PartitionPlanException("Current plan is required"); + } + PartitionPlanValidator.validate(current); + if (StringUtils.isBlank(pluginId) || partitionCount < 1 || StringUtils.isBlank(updatedBy)) { + throw new PartitionPlanException("pluginId, partitionCount, and updatedBy are required"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java new file mode 100644 index 00000000000..7ab09b8e8de --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrap.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Builds the initial bootstrap plan from legacy XML and seeds the registry when the plan topic is empty. */ +public class PartitionPlanBootstrap { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanBootstrap.class); + + private PartitionPlanBootstrap() { + } + + /** Builds the initial bootstrap plan ({@link PartitionPlanConstants#INITIAL_PLAN_VERSION}) using the same contiguous layout as static {@code AuditPartitioner}. */ + public static PartitionPlan createInitialPlan(PartitionPlanBootstrapConfig config) { + if (config == null || StringUtils.isBlank(config.getAuditTopic())) { + throw new PartitionPlanException("Audit topic and bootstrap config are required"); + } + + Map plugins = new LinkedHashMap<>(); + int nextPartition = 0; + for (String plugin : config.getConfiguredPlugins()) { + if (StringUtils.isBlank(plugin)) { + continue; + } + int count = config.getPartitionsForPlugin(plugin.trim()); + plugins.put(plugin.trim(), PluginPartitionAssignment.ofRange(nextPartition, nextPartition + count - 1)); + nextPartition += count; + } + + int topicPartitionCount; + if (nextPartition == 0) { + topicPartitionCount = config.getHashBasedTopicPartitionCount(); + } else { + topicPartitionCount = nextPartition + Math.max(1, config.getBufferPartitionCount()); + } + PartitionPlan plan = PartitionPlan.builder() + .topic(config.getAuditTopic()) + .version(PartitionPlanConstants.INITIAL_PLAN_VERSION) + .topicPartitionCount(topicPartitionCount) + .updatedAt(Instant.now().toString()) + .updatedBy(PartitionPlanConstants.BOOTSTRAP_UPDATED_BY) + .plugins(plugins) + .buffer(PluginPartitionAssignment.ofRange(nextPartition, topicPartitionCount - 1)) + .services(ServiceAllowlistBootstrap.loadAllowlistsFromServerConfig()) + .build(); + + PartitionPlanValidator.validate(plan); + return plan; + } + + /** Builds the first plan from legacy ingestor producer/XML configuration. */ + public static PartitionPlan createInitialPlanFromProducerConfig(Map producerConfig, String auditTopic) { + return createInitialPlan(PartitionPlanBootstrapConfig.fromProducerConfigMap(producerConfig, auditTopic)); + } + + /** + * Empty-registry bootstrap: read registry, build the initial bootstrap plan from XML when no plan + * exists, re-read before publish (concurrent pods), write once, then mandatory read-back. + */ + public static PartitionPlan bootstrapIfEmpty(PartitionPlanRegistry registry, String auditTopic, Map producerConfig) { + PartitionPlan plan = registry.readPlan(auditTopic); + if (plan != null) { + LOG.info("Partition plan version {} already present for audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } + + PartitionPlan localPlan = createInitialPlanFromProducerConfig(producerConfig, auditTopic); + plan = registry.readPlan(auditTopic); + if (plan != null) { + LOG.info("Peer published partition plan version {} while bootstrapping audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } + + registry.writePlan(auditTopic, localPlan); + plan = registry.readPlan(auditTopic); + if (plan == null) { + LOG.error("Mandatory read-back failed after publishing initial bootstrap plan for audit topic '{}'", auditTopic); + throw new PartitionPlanException("Mandatory read-back failed after publishing bootstrap plan for audit topic '" + auditTopic + "'"); + } + LOG.info("Bootstrap partition plan version {} published and read back for audit topic '{}'", plan.getVersion(), auditTopic); + return plan; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java new file mode 100644 index 00000000000..349cf5deff4 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapConfig.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.server.AuditServerConstants; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Inputs for building the first partition plan from legacy ingestor XML / producer config. */ +public class PartitionPlanBootstrapConfig { + private final String auditTopic; + private final String[] configuredPlugins; + private final int defaultPartitionsPerPlugin; + private final int bufferPartitionCount; + private final int hashBasedTopicPartitionCount; + private final Map pluginPartitionOverrides; + + public PartitionPlanBootstrapConfig(String auditTopic, String[] configuredPlugins, int defaultPartitionsPerPlugin, int bufferPartitionCount, int hashBasedTopicPartitionCount, Map pluginPartitionOverrides) { + this.auditTopic = auditTopic; + this.configuredPlugins = configuredPlugins != null ? configuredPlugins : new String[0]; + this.defaultPartitionsPerPlugin = defaultPartitionsPerPlugin; + this.bufferPartitionCount = bufferPartitionCount; + this.hashBasedTopicPartitionCount = Math.max(1, hashBasedTopicPartitionCount); + this.pluginPartitionOverrides = pluginPartitionOverrides != null ? new LinkedHashMap<>(pluginPartitionOverrides) : Collections.emptyMap(); + } + + public static PartitionPlanBootstrapConfig create(String auditTopic, String[] configuredPlugins, int defaultPartitionsPerPlugin, int bufferPartitionCount) { + return new PartitionPlanBootstrapConfig(auditTopic, configuredPlugins, defaultPartitionsPerPlugin, bufferPartitionCount, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS, Collections.emptyMap()); + } + + public PartitionPlanBootstrapConfig withPluginOverride(String pluginId, int partitionCount) { + Map overrides = new LinkedHashMap<>(pluginPartitionOverrides); + overrides.put(pluginId, partitionCount); + return new PartitionPlanBootstrapConfig(auditTopic, configuredPlugins, defaultPartitionsPerPlugin, bufferPartitionCount, hashBasedTopicPartitionCount, overrides); + } + + public String getAuditTopic() { + return auditTopic; + } + + public String[] getConfiguredPlugins() { + return configuredPlugins; + } + + public int getBufferPartitionCount() { + return bufferPartitionCount; + } + + /** Used when {@link #getConfiguredPlugins()} is empty: matches {@code kafka.topic.partitions} / hash-based mode. */ + public int getHashBasedTopicPartitionCount() { + return hashBasedTopicPartitionCount; + } + + public int getPartitionsForPlugin(String pluginId) { + Integer override = pluginPartitionOverrides.get(pluginId); + int count = override != null ? override : defaultPartitionsPerPlugin; + return Math.max(1, count); + } + + public static PartitionPlanBootstrapConfig fromProducerConfigMap(Map configs, String auditTopic) { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + String pluginsStr = getString(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); + String[] plugins = parsePluginIds(pluginsStr); + int defaultPerPlugin = getInt(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, AuditServerConstants.DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN); + int bufferCount = getInt(configs, propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, AuditServerConstants.DEFAULT_BUFFER_PARTITIONS); + int hashBasedTopicPartitions = getInt(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS, AuditServerConstants.DEFAULT_TOPIC_PARTITIONS); + + PartitionPlanBootstrapConfig config = new PartitionPlanBootstrapConfig(auditTopic, plugins, defaultPerPlugin, bufferCount, hashBasedTopicPartitions, Collections.emptyMap()); + for (String plugin : plugins) { + String overrideKey = propPrefix + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin; + if (configs.containsKey(overrideKey)) { + config = config.withPluginOverride(plugin, getInt(configs, overrideKey, defaultPerPlugin)); + } + } + return config; + } + + private static String[] parsePluginIds(String pluginsStr) { + if (pluginsStr == null || pluginsStr.isBlank()) { + return new String[0]; + } + List plugins = new ArrayList<>(); + for (String plugin : pluginsStr.split(",")) { + if (plugin != null && !plugin.isBlank()) { + plugins.add(plugin.trim()); + } + } + return plugins.toArray(new String[0]); + } + + private static String getString(Map configs, String key, String defaultValue) { + Object val = configs.get(key); + return val != null ? val.toString() : defaultValue; + } + + private static int getInt(Map configs, String key, int defaultValue) { + Object val = configs.get(key); + if (val == null) { + return defaultValue; + } + if (val instanceof Number) { + return ((Number) val).intValue(); + } + try { + return Integer.parseInt(val.toString()); + } catch (NumberFormatException e) { + return defaultValue; + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java new file mode 100644 index 00000000000..c89a391b40c --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** Hot-path in-memory plan for {@code AuditPartitioner} and the background watcher. */ +public class PartitionPlanHolder { + private static final PartitionPlanHolder INSTANCE = new PartitionPlanHolder(); + + private final AtomicReference planRef = new AtomicReference<>(); + private volatile int lastInstalledVersion; + + private PartitionPlanHolder() { + } + + public static PartitionPlanHolder getInstance() { + return INSTANCE; + } + + public PartitionPlan getPlan() { + return planRef.get(); + } + + public int getLastInstalledVersion() { + return lastInstalledVersion; + } + + /** Validates and atomically installs the plan used by the Kafka partitioner. */ + public void install(PartitionPlan plan, Integer kafkaPartitionCount) { + PartitionPlanValidator.validate(plan, kafkaPartitionCount); + planRef.set(plan); + lastInstalledVersion = plan.getVersion(); + AuthToLocalRuleComposer.getInstance().applyForPlan(plan); + } + + /** + * Returns allowed short usernames for a service repo from the in-memory registry document. + * {@code null} when the plan has no {@code services} block, or when the repo is not present + * in the plan (caller should fall back to static XML). + * Returns an empty set when the repo is present with an explicit empty allowlist (deny all). + * + *

Used by {@link ServiceAllowlistResolver} for per-repo POST authorization — not for the global + * allowlist union ({@link AuthToLocalRuleComposer#collectAllowedUserShortNames}). + */ + public Set getAllowedUsersForService(String serviceName) { + PartitionPlan plan = planRef.get(); + if (plan == null || plan.getServices() == null || plan.getServices().isEmpty()) { + return null; + } + ServiceAllowlistEntry entry = plan.getServices().get(serviceName); + if (entry == null) { + return null; + } + if (entry.getAllowedUsers().isEmpty()) { + return Collections.emptySet(); + } + return Collections.unmodifiableSet(new LinkedHashSet<>(entry.getAllowedUsers())); + } + + /** Clears holder state between unit tests. */ + public void resetForTests() { + planRef.set(null); + lastInstalledVersion = 0; + AuthToLocalRuleComposer.getInstance().resetForTests(); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java new file mode 100644 index 00000000000..486ed2ce516 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** Kafka client settings for the partition-plan registry topic. */ +public class PartitionPlanKafkaConfig { + private PartitionPlanKafkaConfig() { + } + + /** Resolves the compacted Kafka topic that stores the partition plan registry. */ + public static String resolvePlanTopicName(Properties props, String propPrefix) { + return MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + } + + /** Returns whether ingestor should load routing from the Kafka plan registry. */ + public static boolean isDynamicPartitionPlanEnabled(Properties props, String propPrefix) { + return MiscUtil.getBooleanProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, false); + } + + /** Resolves short usernames allowed to call partition-plan admin REST (empty = any authenticated principal). */ + public static Set resolvePartitionPlanAdminUsers(Properties props, String propPrefix) { + String configured = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_ALLOWED_USERS, ""); + if (configured == null || configured.isBlank()) { + return Collections.emptySet(); + } + Set users = new LinkedHashSet<>(); + for (String user : configured.split(",")) { + if (user != null && !user.isBlank()) { + users.add(user.trim()); + } + } + return users; + } + + /** Returns whether the Kafka producer partitioner should use the in-memory dynamic plan. */ + public static boolean isDynamicPartitionPlanEnabled(Map configs, String ingestorPropPrefix) { + String key = ingestorPropPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED; + Object val = configs.get(key); + if (val == null) { + return false; + } + if (val instanceof Boolean) { + return (Boolean) val; + } + return Boolean.parseBoolean(val.toString()); + } + + /** Resolves how often each ingestor pod reloads the plan from Kafka. */ + public static int resolveRefreshIntervalMs(Properties props, String propPrefix) { + return MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_REFRESH_INTERVAL_MS, AuditServerConstants.DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS); + } + + /** Resolves the Kafka consumer poll timeout when draining the compacted plan topic. */ + public static int resolveConsumerPollTimeoutMs(Properties props, String propPrefix) { + return MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, AuditServerConstants.DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS); + } + + /** Builds Kafka producer properties for writing to the plan registry topic. */ + public static Properties producerConfig(Properties props, String propPrefix) throws Exception { + Properties producerProps = new Properties(); + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); + producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, AuditServerConstants.DEFAULT_PRODUCER_REQUEST_TIMEOUT_MS)); + applySecurity(producerProps, props, propPrefix); + return producerProps; + } + + /** Builds Kafka consumer properties for reading the plan registry topic. */ + public static Properties consumerConfig(Properties props, String propPrefix, String groupId) throws Exception { + Properties consumerProps = new Properties(); + consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS)); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); + consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProps.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, MiscUtil.getIntProperty(props, propPrefix + "." + AuditServerConstants.PROP_REQ_TIMEOUT_MS, AuditServerConstants.DEFAULT_PRODUCER_REQUEST_TIMEOUT_MS)); + applySecurity(consumerProps, props, propPrefix); + return consumerProps; + } + + /** Applies Kerberos/SASL settings shared by plan-registry Kafka clients. */ + private static void applySecurity(Properties clientProps, Properties props, String propPrefix) throws Exception { + String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); + clientProps.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, securityProtocol); + clientProps.put(AuditServerConstants.PROP_SASL_MECHANISM, MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SASL_MECHANISM, AuditServerConstants.DEFAULT_SASL_MECHANISM)); + clientProps.put(AuditServerConstants.PROP_SASL_KERBEROS_SERVICE_NAME, AuditServerConstants.DEFAULT_SERVICE_NAME); + if (securityProtocol.toUpperCase().contains(AuditServerConstants.PROP_SECURITY_PROTOCOL_VALUE)) { + clientProps.put(AuditServerConstants.PROP_SASL_JAAS_CONFIG, AuditMessageQueueUtils.getJAASConfig(props, propPrefix)); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java new file mode 100644 index 00000000000..4d2c190b437 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistry.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; + +/** Durable store for partition plans (Kafka compacted topic in dynamic mode). */ +public interface PartitionPlanRegistry extends AutoCloseable { + PartitionPlan readPlan(String auditTopicKey); + + void writePlan(String auditTopicKey, PartitionPlan plan); + + @Override + void close(); +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java new file mode 100644 index 00000000000..2b07cdc3dc3 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRegistryFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import java.util.Properties; + +/** Opens a Kafka-backed {@link PartitionPlanRegistry} for REST mutations. */ +public class PartitionPlanRegistryFactory { + /** Creates a registry connected to the compacted plan topic. */ + public PartitionPlanRegistry open(Properties props, String propPrefix) throws Exception { + return new KafkaPartitionPlanRegistry(props, propPrefix); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java new file mode 100644 index 00000000000..f4901e35006 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; + +import java.util.List; + +/** Validates partition-plan REST mutation request bodies before registry writes. */ +public class PartitionPlanRequestValidator { + private PartitionPlanRequestValidator() { + } + + public static void validatePatchRequest(PartitionPlanReplacement partitionPlanUpdate) { + if (partitionPlanUpdate == null) { + throw new PartitionPlanException("Partition plan patch request is required"); + } + validateExpectedVersion(partitionPlanUpdate.getExpectedVersion()); + if (!partitionPlanUpdate.hasMergeDelta()) { + throw new PartitionPlanException( + "At least one of topicPartitionCount, plugins, buffer, or services must be provided"); + } + } + + public static void validatePromotePlugin(PromotePlugin promotePluginRequest) { + if (promotePluginRequest == null) { + throw new PartitionPlanException("Promote plugin request is required"); + } + validateNonBlankPluginId(promotePluginRequest.getPluginId()); + validatePositiveCount(promotePluginRequest.getPartitionCount(), "partitionCount"); + validateExpectedVersion(promotePluginRequest.getExpectedVersion()); + if (StringUtils.isNotBlank(promotePluginRequest.getRepo()) + && (promotePluginRequest.getAllowedUsers() == null + || promotePluginRequest.getAllowedUsers().isEmpty())) { + throw new PartitionPlanException("allowedUsers are required when repo is specified"); + } + } + + public static void validateScalePlugin(String pluginId, PluginScale scalePlugin) { + if (scalePlugin == null) { + throw new PartitionPlanException("Plugin scale request is required"); + } + validateNonBlankPluginId(pluginId); + validatePositiveCount(scalePlugin.getAdditionalPartitions(), "additionalPartitions"); + validateExpectedVersion(scalePlugin.getExpectedVersion()); + } + + public static void validateOnboardService(OnboardService onboardServiceRequest) { + if (onboardServiceRequest == null) { + throw new PartitionPlanException("Onboard service request is required"); + } + validateNonBlankServiceName(onboardServiceRequest.getServiceName()); + validateNonBlankPluginId(onboardServiceRequest.getPluginId()); + validatePositiveCount(onboardServiceRequest.getPartitionCount(), "partitionCount"); + validateNonEmptyAllowedUsers(onboardServiceRequest.getAllowedUsers()); + validateExpectedVersion(onboardServiceRequest.getExpectedVersion()); + } + + private static void validateExpectedVersion(int expectedVersion) { + if (expectedVersion < PartitionPlanConstants.INITIAL_PLAN_VERSION) { + throw new PartitionPlanException("expectedVersion must be >= " + PartitionPlanConstants.INITIAL_PLAN_VERSION); + } + } + + private static void validateNonBlankPluginId(String pluginId) { + if (StringUtils.isBlank(pluginId)) { + throw new PartitionPlanException("pluginId is required"); + } + } + + private static void validateNonBlankServiceName(String serviceName) { + if (StringUtils.isBlank(serviceName)) { + throw new PartitionPlanException("serviceName is required"); + } + } + + private static void validatePositiveCount(int count, String fieldName) { + if (count < 1) { + throw new PartitionPlanException(fieldName + " must be >= 1"); + } + } + + private static void validateNonEmptyAllowedUsers(List allowedUsers) { + if (allowedUsers == null || allowedUsers.isEmpty()) { + throw new PartitionPlanException("allowedUsers are required"); + } + boolean hasNonBlankUser = false; + for (String allowedUserShortName : allowedUsers) { + if (StringUtils.isNotBlank(allowedUserShortName)) { + hasNonBlankUser = true; + break; + } + } + if (!hasNonBlankUser) { + throw new PartitionPlanException("allowedUsers must contain at least one non-blank username"); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java new file mode 100644 index 00000000000..53c31b0f47c --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.springframework.stereotype.Component; + +import java.util.Properties; +import java.util.Set; + +/** REST mutations and reads for the dynamic Kafka partition plan. */ +@Component +public class PartitionPlanService { + public static final String INGESTOR_PROP_PREFIX = "ranger.audit.ingestor"; + + private final Properties configProps; + private final PartitionPlanHolder holder; + private final PartitionPlanRegistryFactory registryFactory; + private final KafkaAuditTopicPartitionGrower auditTopicPartitionGrower; + + public PartitionPlanService() { + this(AuditServerConfig.getInstance().getProperties(), PartitionPlanHolder.getInstance(), new PartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + } + + PartitionPlanService(Properties configProps, PartitionPlanHolder holder, PartitionPlanRegistryFactory registryFactory, KafkaAuditTopicPartitionGrower auditTopicPartitionGrower) { + this.configProps = configProps; + this.holder = holder; + this.registryFactory = registryFactory; + this.auditTopicPartitionGrower = auditTopicPartitionGrower; + } + + /** Returns whether dynamic partition-plan mode is enabled in ingestor configuration. */ + public boolean isDynamicPartitionPlanEnabled() { + return PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configProps, INGESTOR_PROP_PREFIX); + } + + /** Returns the plan currently installed in memory on this ingestor pod. */ + public PartitionPlan getPartitionPlan() { + PartitionPlan plan = holder.getPlan(); + if (plan == null) { + throw new PartitionPlanException("Partition plan is not loaded in memory"); + } + return plan; + } + + /** Merges a partial plan delta via REST PATCH with optimistic locking. */ + public PartitionPlan mergePartitionPlan(PartitionPlanReplacement partitionPlanUpdate, String updatedBy) { + PartitionPlanRequestValidator.validatePatchRequest(partitionPlanUpdate); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + requireExpectedVersion(currentPlan, partitionPlanUpdate.getExpectedVersion()); + PartitionPlan mergedPlan = partitionPlanUpdate.toMergedPlan(currentPlan, updatedBy); + if (currentPlan.sameContentAs(mergedPlan)) { + return returnCurrentPlanNoOp(currentPlan); + } + PartitionPlan nextPlan = PartitionPlanAllocator.replacePlan(currentPlan, mergedPlan); + return publishMutation(registry, auditTopic, partitionPlanUpdate.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Promotes a plugin from the buffer to dedicated partitions. */ + public PartitionPlan promotePlugin(PromotePlugin promotePluginRequest, String updatedBy) { + PartitionPlanRequestValidator.validatePromotePlugin(promotePluginRequest); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + requireExpectedVersion(currentPlan, promotePluginRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isPromoteAlreadyApplied(currentPlan, promotePluginRequest.getPluginId(), promotePluginRequest.getPartitionCount(), promotePluginRequest.getRepo(), promotePluginRequest.getAllowedUsers())) { + return returnCurrentPlanNoOp(currentPlan); + } + PartitionPlan nextPlan = PartitionPlanAllocator.promotePlugin(currentPlan, promotePluginRequest.getPluginId(), promotePluginRequest.getPartitionCount(), updatedBy, promotePluginRequest.getRepo(), promotePluginRequest.getAllowedUsers()); + return publishMutation(registry, auditTopic, promotePluginRequest.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Onboards a service repo: upsert allowlist and promote plugin partitions in one plan version. */ + public PartitionPlan onboardService(OnboardService onboardServiceRequest, String updatedBy) { + PartitionPlanRequestValidator.validateOnboardService(onboardServiceRequest); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + requireExpectedVersion(currentPlan, onboardServiceRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isOnboardAlreadyApplied(currentPlan, onboardServiceRequest.getServiceName(), onboardServiceRequest.getPluginId(), onboardServiceRequest.getPartitionCount(), onboardServiceRequest.getAllowedUsers())) { + return returnCurrentPlanNoOp(currentPlan); + } + PartitionPlan nextPlan = PartitionPlanAllocator.onboardRepo(currentPlan, onboardServiceRequest.getServiceName(), onboardServiceRequest.getPluginId(), onboardServiceRequest.getPartitionCount(), onboardServiceRequest.getAllowedUsers(), updatedBy); + return publishMutation(registry, auditTopic, onboardServiceRequest.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to onboard repo in partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Returns configured admin short usernames for partition-plan REST (empty = not restricted beyond authentication). */ + public Set getPartitionPlanAdminUsers() { + return PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); + } + + /** Appends tail partitions to a plugin already present in the plan. */ + public PartitionPlan scalePlugin(String pluginId, PluginScale scalePlugin, String updatedBy) { + PartitionPlanRequestValidator.validateScalePlugin(pluginId, scalePlugin); + requireDynamicEnabled(); + String auditTopic = resolveAuditTopicName(); + try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { + PartitionPlan currentPlan = requirePlan(registry, auditTopic); + PartitionPlan nextPlan = PartitionPlanAllocator.scalePlugin(currentPlan, pluginId, scalePlugin.getAdditionalPartitions(), updatedBy); + return publishMutation(registry, auditTopic, scalePlugin.getExpectedVersion(), currentPlan, nextPlan); + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); + } + } + + /** Validates version, grows the audit topic if needed, writes the plan, and reloads memory. */ + private PartitionPlan publishMutation(PartitionPlanRegistry registry, String auditTopic, int expectedVersion, PartitionPlan current, PartitionPlan next) { + requireExpectedVersion(current, expectedVersion); + growAuditTopicIfNeeded(next.getTopicPartitionCount()); + verifyVersionUnchanged(registry, expectedVersion); + registry.writePlan(auditTopic, next); + verifyReadback(registry, auditTopic, next.getVersion()); + return holder.getPlan(); + } + + /** Returns the current plan without a registry write when the desired state is already satisfied. */ + private PartitionPlan returnCurrentPlanNoOp(PartitionPlan current) { + holder.install(current, current.getTopicPartitionCount()); + return holder.getPlan(); + } + + private static void requireExpectedVersion(PartitionPlan current, int expectedVersion) { + if (current.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(current); + } + } + + /** Grows the audit topic before the plan references new partition IDs. */ + private void growAuditTopicIfNeeded(int requiredPartitions) { + try { + auditTopicPartitionGrower.growAuditTopicToRequiredPartitionCount(configProps, INGESTOR_PROP_PREFIX, resolveAuditTopicName(), requiredPartitions); + } catch (RuntimeException e) { + throw new PartitionPlanException("Failed to grow audit topic partition count", e); + } + } + + /** Confirms the registry still holds the expected version before writing. */ + private void verifyVersionUnchanged(PartitionPlanRegistry registry, int expectedVersion) { + PartitionPlan latest = registry.readPlan(resolveAuditTopicName()); + if (latest == null) { + throw new PartitionPlanException("Partition plan disappeared during update"); + } + if (latest.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(latest); + } + } + + /** Mandatory read-back after publish so every pod converges on the same plan version. */ + private void verifyReadback(PartitionPlanRegistry registry, String auditTopic, int expectedVersion) { + PartitionPlan readback = registry.readPlan(auditTopic); + if (readback == null || readback.getVersion() != expectedVersion) { + throw new PartitionPlanConflictException(readback != null ? readback : holder.getPlan()); + } + holder.install(readback, readback.getTopicPartitionCount()); + } + + /** Loads the current plan from Kafka or fails when the registry is empty. */ + private static PartitionPlan requirePlan(PartitionPlanRegistry registry, String auditTopic) { + PartitionPlan plan = registry.readPlan(auditTopic); + if (plan == null) { + throw new PartitionPlanException("No partition plan found in Kafka for audit topic '" + auditTopic + "'"); + } + return plan; + } + + /** Rejects REST calls when dynamic mode is disabled. */ + private void requireDynamicEnabled() { + if (!isDynamicPartitionPlanEnabled()) { + throw new PartitionPlanException("Dynamic partition plan is not enabled"); + } + } + + /** Resolves the audit data topic name from ingestor configuration. */ + private String resolveAuditTopicName() { + return MiscUtil.getStringProperty(configProps, INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, AuditServerConstants.DEFAULT_TOPIC); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java new file mode 100644 index 00000000000..0ec280ecc81 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplier.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +/** Applies compacted partition-plan Kafka records into {@link PartitionPlanHolder}. */ +public class PartitionPlanUpdateApplier { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanUpdateApplier.class); + + @FunctionalInterface + public interface AuditTopicPartitionCountSupplier { + int getPartitionCount() throws Exception; + } + + private final Properties props; + private final String auditTopicKey; + private final PartitionPlanHolder holder; + private final AuditTopicPartitionCountSupplier partitionCountSupplier; + + public PartitionPlanUpdateApplier( + Properties props, + String auditTopicKey, + PartitionPlanHolder holder, + AuditTopicPartitionCountSupplier partitionCountSupplier) { + this.props = props; + this.auditTopicKey = auditTopicKey; + this.holder = holder != null ? holder : PartitionPlanHolder.getInstance(); + this.partitionCountSupplier = partitionCountSupplier; + } + + /** Installs a plan record when its version is newer than the in-memory copy. */ + public void applyRecordIfNewer(ConsumerRecord record) { + if (!auditTopicKey.equals(record.key())) { + return; + } + try { + PartitionPlan plan = PartitionPlan.fromJson(record.value()); + plan = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + if (plan.getVersion() <= holder.getLastInstalledVersion()) { + return; + } + holder.install(plan, partitionCountSupplier.getPartitionCount()); + LOG.info("Installed partition plan version {} from Kafka offset {}", plan.getVersion(), record.offset()); + } catch (Exception e) { + LOG.error("Ignoring invalid partition plan at offset {} for audit topic '{}'", record.offset(), auditTopicKey, e); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java new file mode 100644 index 00000000000..163c4161057 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Checks partition plan shape and append-only updates. */ +public class PartitionPlanValidator { + private PartitionPlanValidator() { + } + + public static void validate(PartitionPlan plan) { + validate(plan, null); + } + + /** When kafkaPartitionCount is set, it must match plan.topicPartitionCount. */ + public static void validate(PartitionPlan plan, Integer kafkaPartitionCount) { + if (plan == null || StringUtils.isBlank(plan.getTopic()) || plan.getVersion() < PartitionPlanConstants.INITIAL_PLAN_VERSION || plan.getTopicPartitionCount() < 1) { + throw new PartitionPlanException("Invalid partition plan"); + } + if (kafkaPartitionCount != null && !kafkaPartitionCount.equals(plan.getTopicPartitionCount())) { + throw new PartitionPlanException("topicPartitionCount does not match Kafka topic partition count"); + } + + Set assigned = new HashSet<>(); + registerPartitions(plan.getBuffer().getPartitions(), plan.getTopicPartitionCount(), assigned, true); + for (Map.Entry entry : plan.getPlugins().entrySet()) { + if (StringUtils.isBlank(entry.getKey())) { + throw new PartitionPlanException("Plugin id is required"); + } + registerPartitions(entry.getValue().getPartitions(), plan.getTopicPartitionCount(), assigned, false); + } + if (assigned.size() != plan.getTopicPartitionCount()) { + throw new PartitionPlanException("Partition plan must assign every topic partition exactly once"); + } + validateServices(plan.getServices()); + } + + /** When present, each service entry must declare at least one allowed short username. */ + public static void validateServices(Map services) { + if (services == null || services.isEmpty()) { + return; + } + for (Map.Entry entry : services.entrySet()) { + if (StringUtils.isBlank(entry.getKey())) { + throw new PartitionPlanException("Service repo name is required"); + } + ServiceAllowlistEntry allowlistEntry = entry.getValue(); + if (allowlistEntry == null || allowlistEntry.getAllowedUsers().isEmpty()) { + throw new PartitionPlanException("allowedUsers must not be empty for service '" + entry.getKey() + "'"); + } + } + } + + /** New plan must only add tail partitions; existing plugin lists stay unchanged in order. */ + public static void validateAppendOnly(PartitionPlan current, PartitionPlan proposed) { + if (current == null || proposed == null) { + throw new PartitionPlanException("Current and proposed plans are required"); + } + if (proposed.getTopicPartitionCount() < current.getTopicPartitionCount() || proposed.getVersion() != current.getVersion() + 1) { + throw new PartitionPlanException("Plan must grow partition count and increment version by one"); + } + + for (Map.Entry entry : current.getPlugins().entrySet()) { + String pluginId = entry.getKey(); + List before = entry.getValue().getPartitions(); + PluginPartitionAssignment afterAssignment = proposed.getPlugins().get(pluginId); + if (afterAssignment == null) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "'"); + } + List after = afterAssignment.getPartitions(); + if (after.size() < before.size()) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "'"); + } + for (int i = 0; i < before.size(); i++) { + if (!before.get(i).equals(after.get(i))) { + throw new PartitionPlanException("Append-only violation for plugin '" + pluginId + "' at index " + i); + } + } + } + } + + private static void registerPartitions(List partitionIds, int topicPartitionCount, Set assigned, boolean allowEmpty) { + if (partitionIds.isEmpty()) { + if (allowEmpty) { + return; + } + throw new PartitionPlanException("Plugin partition list must not be empty"); + } + for (int partitionId : partitionIds) { + if (partitionId < 0 || partitionId >= topicPartitionCount || !assigned.add(partitionId)) { + throw new PartitionPlanException("Invalid or duplicate partition id: " + partitionId); + } + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java new file mode 100644 index 00000000000..18ed9144476 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanWatcher.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +/** Background thread: load plan from Kafka (or XML-seeded initial bootstrap plan), then incrementally refresh in-memory plan. */ +public class PartitionPlanWatcher implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(PartitionPlanWatcher.class); + + private final Properties props; + private final String propPrefix; + private final String auditTopicKey; + private final PartitionPlanHolder partitionPlanHolder; + private final int refreshIntervalMs; + private final int consumerPollTimeoutMs; + private final String planTopic; + + private PartitionPlanRegistry registry; + private KafkaConsumer consumer; + private PartitionPlanUpdateApplier planUpdateApplier; + private Thread watcherThread; + private volatile boolean running; + + public PartitionPlanWatcher(Properties props, String propPrefix, String auditTopicKey, PartitionPlanHolder holder) { + this.props = props; + this.propPrefix = propPrefix; + this.auditTopicKey = auditTopicKey; + this.partitionPlanHolder = holder != null ? holder : PartitionPlanHolder.getInstance(); + this.refreshIntervalMs = PartitionPlanKafkaConfig.resolveRefreshIntervalMs(props, propPrefix); + this.consumerPollTimeoutMs = PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, propPrefix); + this.planTopic = PartitionPlanKafkaConfig.resolvePlanTopicName(props, propPrefix); + this.planUpdateApplier = new PartitionPlanUpdateApplier(props, auditTopicKey, this.partitionPlanHolder, this::resolveAuditTopicPartitionCount); + } + + /** Blocking startup: empty-registry bootstrap, install plan, then start background refresh. */ + public void startBlocking() throws Exception { + LOG.info("Starting partition plan watcher for audit topic '{}' (plan topic '{}', refresh {} ms, consumer poll {} ms)", auditTopicKey, planTopic, refreshIntervalMs, consumerPollTimeoutMs); + registry = new KafkaPartitionPlanRegistry(props, propPrefix); + Map producerConfig = buildProducerConfigMap(); + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, auditTopicKey, producerConfig); + plan = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + int kafkaPartitionCount = resolveAuditTopicPartitionCount(); + partitionPlanHolder.install(plan, kafkaPartitionCount); + openConsumerAtBeginning(); + pollIncrementalUpdates(); + running = true; + watcherThread = new Thread(this, "PartitionPlanWatcher"); + watcherThread.setDaemon(true); + watcherThread.start(); + LOG.info("Partition plan watcher ready: version={}, auditTopicPartitions={}", plan.getVersion(), kafkaPartitionCount); + } + + /** Stops the background refresh thread and closes Kafka clients. */ + public void stop() { + running = false; + if (watcherThread != null) { + watcherThread.interrupt(); + try { + watcherThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + watcherThread = null; + } + closeConsumer(); + if (registry != null) { + registry.close(); + registry = null; + } + LOG.info("Partition plan watcher stopped"); + } + + @Override + public void run() { + while (running) { + try { + pollIncrementalUpdates(); + Thread.sleep(refreshIntervalMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LOG.error("Partition plan refresh failed; keeping last known good plan version {}", partitionPlanHolder.getLastInstalledVersion(), e); + } + } + } + + /** Drains new compacted plan records and installs any newer version into memory. */ + private void pollIncrementalUpdates() { + if (consumer == null) { + return; + } + ConsumerRecords records; + do { + records = consumer.poll(Duration.ofMillis(consumerPollTimeoutMs)); + for (ConsumerRecord record : records) { + planUpdateApplier.applyRecordIfNewer(record); + } + } while (!records.isEmpty()); + } + + private void openConsumerAtBeginning() throws Exception { + consumer = new KafkaConsumer<>(PartitionPlanKafkaConfig.consumerConfig(props, propPrefix, PartitionPlanConstants.PLAN_WATCHER_CONSUMER_GROUP)); + TopicPartition partition = new TopicPartition(planTopic, 0); + consumer.assign(Collections.singletonList(partition)); + consumer.seekToBeginning(Collections.singletonList(partition)); + } + + private void closeConsumer() { + if (consumer != null) { + consumer.close(); + consumer = null; + } + } + + private int resolveAuditTopicPartitionCount() throws Exception { + try { + Map adminConfig = AuditMessageQueueUtils.buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + DescribeTopicsResult describeTopicsResult = admin.describeTopics(Collections.singletonList(auditTopicKey)); + TopicDescription topicDescription = describeTopicsResult.values().get(auditTopicKey).get(); + return topicDescription.partitions().size(); + } + } catch (Exception e) { + LOG.error("Failed to resolve partition count for audit topic '{}'", auditTopicKey, e); + throw e; + } + } + + private Map buildProducerConfigMap() { + Map producerConfig = new LinkedHashMap<>(); + String prefix = propPrefix + "."; + for (String key : props.stringPropertyNames()) { + if (key.startsWith(prefix)) { + producerConfig.put(key, props.getProperty(key)); + } + } + return producerConfig; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java new file mode 100644 index 00000000000..43e6b32efa6 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PrimaryCatalogRule.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +/** One primary {@code auth_to_local} catalog rule and its mapped Kerberos short name. */ +public class PrimaryCatalogRule { + final String ruleLine; + final String mappedShortName; + + PrimaryCatalogRule(String ruleLine, String mappedShortName) { + this.ruleLine = ruleLine; + this.mappedShortName = mappedShortName; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java new file mode 100644 index 00000000000..5c57b93947d --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_SUFFIX_ALLOWED_USERS; + +/** Loads service allowlist entries from ingestor site XML for registry bootstrap and brownfield merge. */ +public class ServiceAllowlistBootstrap { + private static final String ALLOWLIST_SOURCE_SITE_XML = "xml-bootstrap"; + + private ServiceAllowlistBootstrap() { + } + + /** Scans {@code ranger.audit.ingestor.service..allowed.users} properties. */ + public static Map loadAllowlistsFromProperties(Properties ingestorProperties) { + Map allowlistEntriesByRepo = new LinkedHashMap<>(); + if (ingestorProperties == null) { + return allowlistEntriesByRepo; + } + for (String propertyName : ingestorProperties.stringPropertyNames()) { + if (!propertyName.startsWith(PROP_PREFIX_AUDIT_SERVER_SERVICE) || !propertyName.endsWith(PROP_SUFFIX_ALLOWED_USERS)) { + continue; + } + String serviceRepoName = propertyName.substring(PROP_PREFIX_AUDIT_SERVER_SERVICE.length(), propertyName.length() - PROP_SUFFIX_ALLOWED_USERS.length()); + if (StringUtils.isBlank(serviceRepoName)) { + continue; + } + String allowedUsersPropertyValue = ingestorProperties.getProperty(propertyName); + if (StringUtils.isBlank(allowedUsersPropertyValue)) { + continue; + } + List allowedUserShortNames = parseAllowedUserShortNames(allowedUsersPropertyValue); + if (allowedUserShortNames.isEmpty()) { + continue; + } + allowlistEntriesByRepo.put( + serviceRepoName.trim(), + new ServiceAllowlistEntry(allowedUserShortNames, ALLOWLIST_SOURCE_SITE_XML, null)); + } + return allowlistEntriesByRepo; + } + + /** Loads allowlist entries from the running ingestor configuration singleton. */ + public static Map loadAllowlistsFromServerConfig() { + return loadAllowlistsFromProperties(AuditServerConfig.getInstance().getProperties()); + } + + /** + * Brownfield helper: when the Kafka plan has no {@code services} block, merge XML entries in memory only + * (does not bump version or write to Kafka). + */ + public static PartitionPlan mergeSiteXmlAllowlistsWhenPlanServicesMissing( + PartitionPlan partitionPlan, Properties ingestorProperties) { + if (partitionPlan == null || (partitionPlan.getServices() != null && !partitionPlan.getServices().isEmpty())) { + return partitionPlan; + } + Map siteXmlAllowlistEntries = loadAllowlistsFromProperties(ingestorProperties); + if (siteXmlAllowlistEntries.isEmpty()) { + return partitionPlan; + } + return partitionPlan.toBuilder().services(siteXmlAllowlistEntries).build(); + } + + private static List parseAllowedUserShortNames(String allowedUsersPropertyValue) { + List allowedUserShortNames = new ArrayList<>(); + for (String userToken : allowedUsersPropertyValue.split(",")) { + if (userToken != null) { + String trimmedShortName = userToken.trim(); + if (StringUtils.isNotBlank(trimmedShortName)) { + allowedUserShortNames.add(trimmedShortName); + } + } + } + return allowedUserShortNames; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java new file mode 100644 index 00000000000..8b0b09bceb5 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.Set; + +/** + * Per-repo audit POST authorization after {@code auth_to_local} mapping. + * + *

Distinct from the global allowlist union in {@link AuthToLocalRuleComposer#collectAllowedUserShortNames} + * (which selects Kerberos mapping rules). Checks whether the mapped short name is allowed for the + * specific {@code serviceName} on {@code POST /api/audit/access?serviceName=...}. + * + *

Example (dynamic mode, plan has dev_hdfs with hdfs and nn only): + *

    + *
  • isAllowed(dev_hdfs, hdfs) returns true (repo in plan)
    • + *
  • isAllowed(dev_hive, hive) returns false unless static XML lists hive for dev_hive + * (repo not in plan -> registry returns null -> XML fallback)
    • + *
+ */ +public class ServiceAllowlistResolver { + private ServiceAllowlistResolver() { + } + + /** + * @param serviceName Ranger repo from {@code serviceName} query param (e.g. {@code dev_hdfs}) + * @param userName short name after {@code KerberosName.getShortName()} (e.g. {@code hdfs} from {@code nn/...}) + * @param holder partition-plan registry; per-repo {@code services[serviceName].allowedUsers} + */ + public static boolean isAllowedServiceUser(String serviceName, String userName, boolean dynamicPartitionPlanEnabled, PartitionPlanHolder holder, Map> staticAllowedUsersByService) { + if (StringUtils.isBlank(serviceName) || StringUtils.isBlank(userName)) { + return false; + } + if (dynamicPartitionPlanEnabled && holder != null) { + Set registryUsers = holder.getAllowedUsersForService(serviceName); + if (registryUsers != null) { + return registryUsers.contains(userName); + } + } + Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; + return allowedUsers != null && allowedUsers.contains(userName); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java new file mode 100644 index 00000000000..13436b34484 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/constants/PartitionPlanConstants.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.constants; + +/** Shared constants for the dynamic Kafka partition-plan registry. */ +public class PartitionPlanConstants { + /** Version number of the first XML-seeded initial bootstrap plan in an empty registry. */ + public static final int INITIAL_PLAN_VERSION = 1; + /** {@code updatedBy} value on plans published by {@code PartitionPlanBootstrap}. */ + public static final String BOOTSTRAP_UPDATED_BY = "bootstrap"; + public static final String PLAN_REGISTRY_CONSUMER_GROUP = "ranger_audit_partition_plan_registry"; + public static final String PLAN_WATCHER_CONSUMER_GROUP = "ranger_audit_partition_plan_watcher"; + + private PartitionPlanConstants() { + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java new file mode 100644 index 00000000000..7d22de9f7fe --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanConflictException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.exception; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; + +/** Optimistic-lock failure: another writer changed the plan version first (HTTP 409). */ +public final class PartitionPlanConflictException extends PartitionPlanException { + private static final long serialVersionUID = 1L; + + private final PartitionPlan currentPlan; + + public PartitionPlanConflictException(PartitionPlan currentPlan) { + super("Partition plan version conflict"); + this.currentPlan = currentPlan; + } + + public PartitionPlan getCurrentPlan() { + return currentPlan; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java new file mode 100644 index 00000000000..9080e4b2382 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/exception/PartitionPlanException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.exception; + +/** + * Raised when a partition plan is invalid or an append-only allocation cannot be applied. + */ +public class PartitionPlanException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public PartitionPlanException(String message) { + super(message); + } + + public PartitionPlanException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java new file mode 100644 index 00000000000..b39331eaa47 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class OnboardService implements Serializable { + private final String serviceName; + private final String pluginId; + private final int partitionCount; + private final List allowedUsers; + private final int expectedVersion; + + @JsonCreator + public OnboardService(@JsonProperty("serviceName") String serviceName, @JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("expectedVersion") int expectedVersion) { + this.serviceName = serviceName; + this.pluginId = pluginId; + this.partitionCount = partitionCount; + this.allowedUsers = allowedUsers == null ? Collections.emptyList() : List.copyOf(allowedUsers); + this.expectedVersion = expectedVersion; + } + + public String getServiceName() { + return serviceName; + } + + public String getPluginId() { + return pluginId; + } + + public int getPartitionCount() { + return partitionCount; + } + + public List getAllowedUsers() { + return allowedUsers; + } + + public int getExpectedVersion() { + return expectedVersion; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java new file mode 100644 index 00000000000..845555bea41 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanValidator; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.provider.MiscUtil; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PartitionPlan implements Serializable { + private final String topic; + private final int version; + private final int topicPartitionCount; + private final String updatedAt; + private final String updatedBy; + private final Map plugins; + private final PluginPartitionAssignment buffer; + private final Map services; + + @JsonCreator + public PartitionPlan(@JsonProperty("topic") String topic, @JsonProperty("version") int version, @JsonProperty("topicPartitionCount") int topicPartitionCount, @JsonProperty("updatedAt") String updatedAt, @JsonProperty("updatedBy") String updatedBy, @JsonProperty("plugins") Map plugins, @JsonProperty("buffer") PluginPartitionAssignment buffer, @JsonProperty("services") Map services) { + this.topic = topic; + this.version = version; + this.topicPartitionCount = topicPartitionCount; + this.updatedAt = updatedAt; + this.updatedBy = updatedBy; + this.plugins = copyPlugins(plugins); + this.buffer = buffer != null ? buffer : PluginPartitionAssignment.empty(); + this.services = copyServices(services); + } + + private static Map copyPlugins(Map plugins) { + if (plugins == null || plugins.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(plugins)); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public String getTopic() { + return topic; + } + + public int getVersion() { + return version; + } + + public int getTopicPartitionCount() { + return topicPartitionCount; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUpdatedBy() { + return updatedBy; + } + + public Map getPlugins() { + return plugins; + } + + public PluginPartitionAssignment getBuffer() { + return buffer; + } + + public Map getServices() { + return services; + } + + /** Compares routing payload; ignores version, updatedAt, and updatedBy. */ + public boolean sameContentAs(PartitionPlan other) { + if (other == null) { + return false; + } + return topicPartitionCount == other.topicPartitionCount + && Objects.equals(topic, other.topic) + && Objects.equals(plugins, other.plugins) + && Objects.equals(buffer, other.buffer) + && Objects.equals(services, other.services); + } + + public Builder toBuilder() { + return new Builder(this); + } + + public static Builder builder() { + return new Builder(); + } + + /** Serializes this plan for the compacted Kafka registry topic. */ + public String toJson() { + try { + return MiscUtil.getMapper().writeValueAsString(this); + } catch (Exception e) { + throw new PartitionPlanException("Failed to serialize partition plan", e); + } + } + + /** Parses and validates a plan JSON payload from Kafka or REST. */ + public static PartitionPlan fromJson(String json) { + try { + PartitionPlan plan = MiscUtil.getMapper().readValue(json, PartitionPlan.class); + PartitionPlanValidator.validate(plan); + return plan; + } catch (PartitionPlanException e) { + throw e; + } catch (Exception e) { + throw new PartitionPlanException("Failed to deserialize partition plan", e); + } + } + + @Override + public boolean equals(Object otherPartitionPlanObj) { + if (this == otherPartitionPlanObj) { + return true; + } + if (otherPartitionPlanObj == null || getClass() != otherPartitionPlanObj.getClass()) { + return false; + } + PartitionPlan otherPartitionPlan = (PartitionPlan) otherPartitionPlanObj; + return version == otherPartitionPlan.version + && topicPartitionCount == otherPartitionPlan.topicPartitionCount + && Objects.equals(topic, otherPartitionPlan.topic) + && Objects.equals(updatedAt, otherPartitionPlan.updatedAt) + && Objects.equals(updatedBy, otherPartitionPlan.updatedBy) + && Objects.equals(plugins, otherPartitionPlan.plugins) + && Objects.equals(buffer, otherPartitionPlan.buffer) + && Objects.equals(services, otherPartitionPlan.services); + } + + @Override + public int hashCode() { + return Objects.hash(topic, version, topicPartitionCount, updatedAt, updatedBy, plugins, buffer, services); + } + + @Override + public String toString() { + return "PartitionPlan{topic='" + topic + "', version=" + version + ", topicPartitionCount=" + topicPartitionCount + ", plugins=" + plugins.keySet() + ", bufferSize=" + buffer.size() + ", services=" + services.keySet() + '}'; + } + + public static final class Builder { + private String topic; + private int version = 1; + private int topicPartitionCount; + private String updatedAt; + private String updatedBy; + private Map plugins = new LinkedHashMap<>(); + private PluginPartitionAssignment buffer = PluginPartitionAssignment.empty(); + private Map services = new LinkedHashMap<>(); + + private Builder() { + } + + private Builder(PartitionPlan plan) { + this.topic = plan.topic; + this.version = plan.version; + this.topicPartitionCount = plan.topicPartitionCount; + this.updatedAt = plan.updatedAt; + this.updatedBy = plan.updatedBy; + this.plugins = new LinkedHashMap<>(plan.plugins); + this.buffer = plan.buffer; + this.services = new LinkedHashMap<>(plan.services); + } + + public Builder topic(String topic) { + this.topic = topic; + return this; + } + + public Builder version(int version) { + this.version = version; + return this; + } + + public Builder topicPartitionCount(int topicPartitionCount) { + this.topicPartitionCount = topicPartitionCount; + return this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return this; + } + + public Builder updatedBy(String updatedBy) { + this.updatedBy = updatedBy; + return this; + } + + public Builder plugins(Map plugins) { + this.plugins = plugins == null ? new LinkedHashMap<>() : new LinkedHashMap<>(plugins); + return this; + } + + public Builder putPlugin(String pluginId, PluginPartitionAssignment assignment) { + this.plugins.put(pluginId, assignment); + return this; + } + + public Builder buffer(PluginPartitionAssignment buffer) { + this.buffer = buffer != null ? buffer : PluginPartitionAssignment.empty(); + return this; + } + + public Builder services(Map services) { + this.services = services == null ? new LinkedHashMap<>() : new LinkedHashMap<>(services); + return this; + } + + public PartitionPlan build() { + return new PartitionPlan(topic, version, topicPartitionCount, updatedAt, updatedBy, plugins, buffer, services); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java new file mode 100644 index 00000000000..479377c1b95 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Partial partition-plan update (REST PATCH). Omitted or empty fields are inherited from the current plan. + * {@code plugins} and {@code services} merge only entries whose keys are not already present. + */ +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PartitionPlanReplacement implements Serializable { + private final int expectedVersion; + private final Integer topicPartitionCount; + private final Map plugins; + private final PluginPartitionAssignment buffer; + private final Map services; + + @JsonCreator + public PartitionPlanReplacement(@JsonProperty("expectedVersion") int expectedVersion, + @JsonProperty("topicPartitionCount") Integer topicPartitionCount, + @JsonProperty("plugins") Map plugins, + @JsonProperty("buffer") PluginPartitionAssignment buffer, + @JsonProperty("services") Map services) { + this.expectedVersion = expectedVersion; + this.topicPartitionCount = topicPartitionCount; + this.plugins = copyPlugins(plugins); + this.buffer = buffer; + this.services = copyServices(services); + } + + public PartitionPlanReplacement(int expectedVersion, int topicPartitionCount, + Map plugins, PluginPartitionAssignment buffer, + Map services) { + this(expectedVersion, Integer.valueOf(topicPartitionCount), plugins, buffer, services); + } + + private static Map copyPlugins(Map plugins) { + if (plugins == null || plugins.isEmpty()) { + return null; + } + return Collections.unmodifiableMap(new LinkedHashMap<>(plugins)); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return null; + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public Integer getTopicPartitionCount() { + return topicPartitionCount; + } + + public Map getPlugins() { + return plugins; + } + + public PluginPartitionAssignment getBuffer() { + return buffer; + } + + public Map getServices() { + return services; + } + + /** True when the request carries at least one field to merge into the current plan. */ + public boolean hasMergeDelta() { + return hasTopicPartitionCountDelta() || hasPluginsDelta() || hasBufferDelta() || hasServicesDelta(); + } + + public boolean hasTopicPartitionCountDelta() { + return topicPartitionCount != null && topicPartitionCount >= 1; + } + + public boolean hasPluginsDelta() { + return plugins != null && !plugins.isEmpty(); + } + + public boolean hasBufferDelta() { + return buffer != null && !buffer.getPartitions().isEmpty(); + } + + public boolean hasServicesDelta() { + return services != null && !services.isEmpty(); + } + + /** Merges this delta into {@code currentPlan} and returns the proposed next plan (version not incremented). */ + public PartitionPlan toMergedPlan(PartitionPlan currentPlan, String updatedBy) { + if (currentPlan == null) { + throw new PartitionPlanException("Current partition plan is required"); + } + + int mergedTopicPartitionCount = hasTopicPartitionCountDelta() + ? topicPartitionCount + : currentPlan.getTopicPartitionCount(); + + Map mergedPlugins = new LinkedHashMap<>(currentPlan.getPlugins()); + List newlyAddedPluginAssignments = new ArrayList<>(); + if (hasPluginsDelta()) { + for (Map.Entry pluginDeltaEntry : plugins.entrySet()) { + String pluginId = pluginDeltaEntry.getKey(); + if (StringUtils.isBlank(pluginId)) { + throw new PartitionPlanException("Plugin id is required"); + } + if (mergedPlugins.containsKey(pluginId)) { + PluginPartitionAssignment existingAssignment = mergedPlugins.get(pluginId); + PluginPartitionAssignment requestedAssignment = pluginDeltaEntry.getValue(); + if (existingAssignment.equals(requestedAssignment)) { + continue; + } + throw new PartitionPlanException( + "Plugin '" + pluginId + "' already exists with different partition assignment; use PATCH /partition-plan/plugins/{pluginId} to grow it"); + } + mergedPlugins.put(pluginId, pluginDeltaEntry.getValue()); + newlyAddedPluginAssignments.add(pluginDeltaEntry.getValue()); + } + } + + PluginPartitionAssignment mergedBuffer = hasBufferDelta() ? buffer : currentPlan.getBuffer(); + + Map mergedServices = new LinkedHashMap<>(currentPlan.getServices()); + if (hasServicesDelta()) { + for (Map.Entry serviceDeltaEntry : services.entrySet()) { + String serviceRepoName = serviceDeltaEntry.getKey(); + if (StringUtils.isBlank(serviceRepoName)) { + throw new PartitionPlanException("Service repo name is required"); + } + mergedServices.put(serviceRepoName.trim(), serviceDeltaEntry.getValue()); + } + } + + if (!newlyAddedPluginAssignments.isEmpty()) { + mergedBuffer = removeAssignedPartitionsFromBuffer(mergedBuffer, newlyAddedPluginAssignments); + } + + return PartitionPlan.builder() + .topic(currentPlan.getTopic()) + .topicPartitionCount(mergedTopicPartitionCount) + .plugins(mergedPlugins) + .buffer(mergedBuffer) + .services(mergedServices) + .updatedAt(Instant.now().toString()) + .updatedBy(updatedBy) + .build(); + } + + private static PluginPartitionAssignment removeAssignedPartitionsFromBuffer( + PluginPartitionAssignment bufferAssignment, Iterable newPluginAssignments) { + List remainingBufferPartitionIds = new ArrayList<>(bufferAssignment.getPartitions()); + for (PluginPartitionAssignment pluginAssignment : newPluginAssignments) { + for (Integer partitionId : pluginAssignment.getPartitions()) { + remainingBufferPartitionIds.remove(partitionId); + } + } + return new PluginPartitionAssignment(remainingBufferPartitionIds); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java new file mode 100644 index 00000000000..50157c9d7a3 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginPartitionAssignment.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PluginPartitionAssignment implements java.io.Serializable { + private final List partitions; + + @JsonCreator + public PluginPartitionAssignment(@JsonProperty("partitions") List partitions) { + if (partitions == null || partitions.isEmpty()) { + this.partitions = Collections.emptyList(); + } else { + this.partitions = List.copyOf(partitions); + } + } + + /** Returns an assignment with no partition IDs (empty buffer). */ + public static PluginPartitionAssignment empty() { + return new PluginPartitionAssignment(Collections.emptyList()); + } + + /** Builds an assignment from explicit partition IDs. */ + public static PluginPartitionAssignment of(int... partitionIds) { + List ids = new ArrayList<>(partitionIds.length); + for (int id : partitionIds) { + ids.add(id); + } + return new PluginPartitionAssignment(ids); + } + + /** Builds a contiguous inclusive partition ID range. */ + public static PluginPartitionAssignment ofRange(int startInclusive, int endInclusive) { + if (endInclusive < startInclusive) { + throw new IllegalArgumentException("Invalid partition range: " + startInclusive + "-" + endInclusive); + } + List ids = new ArrayList<>(endInclusive - startInclusive + 1); + for (int i = startInclusive; i <= endInclusive; i++) { + ids.add(i); + } + return new PluginPartitionAssignment(ids); + } + + public List getPartitions() { + return partitions; + } + + public int size() { + return partitions.size(); + } + + @Override + public boolean equals(Object otherPluginPartitionAssignmentObj) { + if (this == otherPluginPartitionAssignmentObj) { + return true; + } + if (otherPluginPartitionAssignmentObj == null || getClass() != otherPluginPartitionAssignmentObj.getClass()) { + return false; + } + PluginPartitionAssignment otherAssignment = (PluginPartitionAssignment) otherPluginPartitionAssignmentObj; + return Objects.equals(partitions, otherAssignment.partitions); + } + + @Override + public int hashCode() { + return Objects.hash(partitions); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java new file mode 100644 index 00000000000..73e493f7b39 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PluginScale implements Serializable { + private final int additionalPartitions; + private final int expectedVersion; + + @JsonCreator + public PluginScale(@JsonProperty("additionalPartitions") int additionalPartitions, @JsonProperty("expectedVersion") int expectedVersion) { + this.additionalPartitions = additionalPartitions; + this.expectedVersion = expectedVersion; + } + + public int getAdditionalPartitions() { + return additionalPartitions; + } + + public int getExpectedVersion() { + return expectedVersion; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java new file mode 100644 index 00000000000..ef8d347d373 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PromotePlugin implements Serializable { + private final String pluginId; + private final int partitionCount; + private final int expectedVersion; + private final String repo; + private final List allowedUsers; + + @JsonCreator + public PromotePlugin(@JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("repo") String repo, @JsonProperty("allowedUsers") List allowedUsers) { + this.pluginId = pluginId; + this.partitionCount = partitionCount; + this.expectedVersion = expectedVersion; + this.repo = repo; + this.allowedUsers = allowedUsers == null ? Collections.emptyList() : List.copyOf(allowedUsers); + } + + public PromotePlugin(String pluginId, int partitionCount, int expectedVersion) { + this(pluginId, partitionCount, expectedVersion, null, null); + } + + public String getPluginId() { + return pluginId; + } + + public int getPartitionCount() { + return partitionCount; + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public String getRepo() { + return repo; + } + + public List getAllowedUsers() { + return allowedUsers; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java new file mode 100644 index 00000000000..b461afde267 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ServiceAllowlistEntry implements Serializable { + private final List allowedUsers; + private final String source; + private final String notes; + + @JsonCreator + public ServiceAllowlistEntry(@JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("source") String source, @JsonProperty("notes") String notes) { + this.allowedUsers = copyAllowedUsers(allowedUsers); + this.source = source; + this.notes = notes; + } + + public static ServiceAllowlistEntry ofUsers(String... users) { + return new ServiceAllowlistEntry(List.of(users), null, null); + } + + public static ServiceAllowlistEntry ofUsers(List users) { + return new ServiceAllowlistEntry(users, null, null); + } + + private static List copyAllowedUsers(List allowedUsers) { + if (allowedUsers == null || allowedUsers.isEmpty()) { + return Collections.emptyList(); + } + Set unique = new LinkedHashSet<>(); + for (String user : allowedUsers) { + if (user != null && !user.isBlank()) { + unique.add(user.trim()); + } + } + return List.copyOf(unique); + } + + public List getAllowedUsers() { + return allowedUsers; + } + + public String getSource() { + return source; + } + + public String getNotes() { + return notes; + } + + /** True when normalized allowedUsers match (ignores source and notes). */ + public boolean hasSameAllowedUsers(List users) { + return Objects.equals(allowedUsers, ofUsers(users).getAllowedUsers()); + } + + @Override + public boolean equals(Object otherServiceAllowlistEntryObj) { + if (this == otherServiceAllowlistEntryObj) { + return true; + } + if (otherServiceAllowlistEntryObj == null || getClass() != otherServiceAllowlistEntryObj.getClass()) { + return false; + } + ServiceAllowlistEntry otherAllowlistEntry = (ServiceAllowlistEntry) otherServiceAllowlistEntryObj; + return Objects.equals(allowedUsers, otherAllowlistEntry.allowedUsers) + && Objects.equals(source, otherAllowlistEntry.source) + && Objects.equals(notes, otherAllowlistEntry.notes); + } + + @Override + public int hashCode() { + return Objects.hash(allowedUsers, source, notes); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 2ae7d14e0f2..3f9b9816982 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -23,6 +23,19 @@ import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.AuditDestinationMgr; +import org.apache.ranger.audit.producer.kafka.partition.AuthToLocalRuleComposer; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanService; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanUpdateApplier; +import org.apache.ranger.audit.producer.kafka.partition.ServiceAllowlistResolver; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; @@ -35,8 +48,10 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.PATCH; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; @@ -68,6 +83,9 @@ public class AuditREST { @Autowired AuditDestinationMgr auditDestinationMgr; + @Autowired + PartitionPlanService partitionPlanService; + /** * Health check endpoint */ @@ -147,7 +165,7 @@ public Response getStatus() { } /** - * Access Audits producer endpoint. + * Access Audits producer endpoint. * @param serviceName Required query parameter to identify the source service (hdfs, hive, kafka, solr, etc.) * @param appId Optional query parameter for batch processing - identifies the application instance * @param accessAudits List of audit events to process @@ -240,6 +258,225 @@ public Response logAccessAudit(@QueryParam("serviceName") String serviceName, @Q return ret; } + /** Returns the in-memory partition plan when dynamic mode is enabled. */ + @GET + @Path("/partition-plan") + @Produces("application/json") + public Response getPartitionPlan(@Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.getPartitionPlan()"); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("GET /partition-plan"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "GET /partition-plan"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = Response.ok(partitionPlanService.getPartitionPlan().toJson()).build(); + } catch (PartitionPlanException e) { + LOG.error("Partition plan GET failed", e); + ret = Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(buildErrorResponse(e.getMessage())).build(); + } + } + } + LOG.debug("<== AuditREST.getPartitionPlan(): status={}", ret.getStatus()); + return ret; + } + + /** Applies a partial update to the partition plan (optimistic lock via expectedVersion). */ + @PATCH + @Path("/partition-plan") + @Consumes("application/json") + @Produces("application/json") + public Response patchPartitionPlan(PartitionPlanReplacement partitionPlanUpdate, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.patchPartitionPlan()"); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("PATCH /partition-plan"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "PATCH /partition-plan"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.mergePartitionPlan(partitionPlanUpdate, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("PATCH /partition-plan", e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("PATCH /partition-plan", e); + } catch (Exception e) { + LOG.error("Unexpected error patching partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to patch partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.patchPartitionPlan(): status={}", ret.getStatus()); + return ret; + } + + /** Promotes a plugin from the buffer to dedicated partitions. */ + @POST + @Path("/partition-plan/plugins") + @Consumes("application/json") + @Produces("application/json") + public Response createPluginAssignment(PromotePlugin request, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.createPluginAssignment(pluginId={})", request != null ? request.getPluginId() : null); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("POST /partition-plan/plugins"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "POST /partition-plan/plugins"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.promotePlugin(request, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("POST /partition-plan/plugins", e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("POST /partition-plan/plugins", e); + } catch (Exception e) { + LOG.error("Unexpected error creating plugin assignment in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to create plugin assignment in partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.createPluginAssignment(): status={}", ret.getStatus()); + return ret; + } + + /** Appends tail partitions to an existing plugin assignment. */ + @PATCH + @Path("/partition-plan/plugins/{pluginId}") + @Consumes("application/json") + @Produces("application/json") + public Response scalePluginAssignment(@PathParam("pluginId") String pluginId, PluginScale pluginScale, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.scalePluginAssignment(pluginId={})", pluginId); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("PATCH /partition-plan/plugins/{pluginId}"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "PATCH /partition-plan/plugins/{pluginId}"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.scalePlugin(pluginId, pluginScale, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("PATCH /partition-plan/plugins/" + pluginId, e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("PATCH /partition-plan/plugins/" + pluginId, e); + } catch (Exception e) { + LOG.error("Unexpected error scaling plugin assignment in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to scale plugin assignment in partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.scalePluginAssignment(): status={}", ret.getStatus()); + return ret; + } + + /** Onboards a service: upsert allowlist and promote plugin partitions in one plan version. */ + @POST + @Path("/partition-plan/services") + @Consumes("application/json") + @Produces("application/json") + public Response createService(OnboardService request, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.createService(serviceName={}, pluginId={})", + request != null ? request.getServiceName() : null, request != null ? request.getPluginId() : null); + Response ret; + if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { + ret = partitionPlanDisabled("POST /partition-plan/services"); + } else { + Response authFailure = authorizePartitionPlanAdmin(httpRequest, "POST /partition-plan/services"); + if (authFailure != null) { + ret = authFailure; + } else { + try { + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.onboardService(request, resolveUpdatedBy(httpRequest))); + } catch (PartitionPlanConflictException e) { + ret = toPartitionPlanConflictResponse("POST /partition-plan/services", e); + } catch (PartitionPlanException e) { + ret = toPartitionPlanErrorResponse("POST /partition-plan/services", e); + } catch (Exception e) { + LOG.error("Unexpected error creating service in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to create service in partition plan")).build(); + } + } + } + LOG.debug("<== AuditREST.createService(): status={}", ret.getStatus()); + return ret; + } + + /** Returns HTTP 200 with the updated plan JSON body. */ + private Response toSuccessfulPartitionPlanResponse(PartitionPlan updatedPlan) { + return Response.ok(updatedPlan.toJson()).build(); + } + + /** Returns HTTP 409 with the current plan when optimistic locking fails. */ + private Response toPartitionPlanConflictResponse(String operation, PartitionPlanConflictException conflict) { + PartitionPlan currentPlan = conflict.getCurrentPlan(); + if (currentPlan == null) { + LOG.error("{} rejected: partition plan version conflict (current plan unavailable)", operation, conflict); + return Response.status(Response.Status.CONFLICT).entity(buildErrorResponse("Partition plan version conflict")).build(); + } + LOG.error("{} rejected: partition plan version conflict; current version is {}", operation, currentPlan.getVersion(), conflict); + return Response.status(Response.Status.CONFLICT).entity(currentPlan.toJson()).build(); + } + + /** Returns HTTP 400 or 503 for validation and Kafka admin failures. */ + private Response toPartitionPlanErrorResponse(String operation, PartitionPlanException error) { + Response.Status status = resolvePartitionPlanErrorStatus(error); + LOG.error("{} failed: {}", operation, error.getMessage(), error); + return Response.status(status).entity(buildErrorResponse(error.getMessage())).build(); + } + + /** Returns HTTP 503 when dynamic partition-plan mode is disabled. */ + private Response partitionPlanDisabled(String operation) { + LOG.error("{} rejected: dynamic partition plan is not enabled", operation); + return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(buildErrorResponse("Dynamic partition plan is not enabled")).build(); + } + + /** + * When {@code kafka.partition.plan.allowed.users} is configured, restrict partition-plan REST to those short names. + * When unset, any authenticated principal may call partition-plan (backward compatible). + */ + private Response authorizePartitionPlanAdmin(HttpServletRequest request, String operation) { + String user = getAuthenticatedUser(request); + if (StringUtils.isBlank(user)) { + LOG.error("{} rejected: authentication required", operation); + return Response.status(Response.Status.UNAUTHORIZED).entity(buildErrorResponse("Authentication required")).build(); + } + Set adminUsers = partitionPlanService.getPartitionPlanAdminUsers(); + if (!adminUsers.isEmpty() && !adminUsers.contains(user)) { + LOG.error("{} rejected: user '{}' is not in partition plan admin allowlist", operation, user); + return Response.status(Response.Status.FORBIDDEN).entity(buildErrorResponse("User is not authorized to manage partition plan")).build(); + } + return null; + } + + /** Maps service/infrastructure failures to 503; client validation mistakes to 400. */ + private static Response.Status resolvePartitionPlanErrorStatus(PartitionPlanException error) { + if (error.getCause() != null) { + return Response.Status.SERVICE_UNAVAILABLE; + } + String message = error.getMessage(); + if (message != null && (message.contains("Partition plan is not loaded in memory") + || message.contains("Partition plan disappeared during update") + || message.contains("No partition plan found in Kafka") + || message.contains("Mandatory read-back failed"))) { + return Response.Status.SERVICE_UNAVAILABLE; + } + return Response.Status.BAD_REQUEST; + } + + /** Records the authenticated admin user on plan mutations. */ + private String resolveUpdatedBy(HttpServletRequest request) { + String user = getAuthenticatedUser(request); + return StringUtils.isNotBlank(user) ? user : "rest-api"; + } + private String buildResponse(Map respMap) { try { return MiscUtil.getMapper().writeValueAsString(respMap); @@ -302,21 +539,27 @@ private String applyAuthToLocal(String principal) { } /** - * Rules are loaded from ranger.audit.ingestor.auth.to.local property in ranger-audit-ingestor-site.xml. + * Loads the auth_to_local catalog from ranger-audit-ingestor-site.xml. When dynamic partition-plan + * mode is disabled, applies the full catalog immediately. When enabled, applies static XML rules + * only if the partition-plan Kafka topic does not exist yet; otherwise composed rules are installed + * on {@link PartitionPlanHolder#install(PartitionPlan, Integer)} via {@link AuthToLocalRuleComposer#applyForPlan}. */ private static void initializeAuthToLocal() { AuditServerConfig config = AuditServerConfig.getInstance(); String authToLocalRules = config.get(AuditServerConstants.PROP_AUTH_TO_LOCAL); - if (StringUtils.isNotEmpty(authToLocalRules)) { - try { - KerberosName.setRules(authToLocalRules); - LOG.debug("Auth_to_local rules: {}", authToLocalRules); - } catch (Exception e) { - LOG.error("Failed to set auth_to_local rules from configuration: {}", e.getMessage(), e); - } - } else { + if (StringUtils.isEmpty(authToLocalRules)) { LOG.warn("No auth_to_local rules configured. Kerberos principal mapping may not work correctly."); LOG.warn("Set property '{}' in ranger-audit-ingestor-site.xml", AuditServerConstants.PROP_AUTH_TO_LOCAL); + return; + } + + AuthToLocalRuleComposer composer = AuthToLocalRuleComposer.getInstance(); + composer.initializeFromConfig(); + if (PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX)) { + composer.applyStartupRulesForDynamicMode(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX); + } else { + composer.applyStaticRules(); + LOG.debug("Applied static auth_to_local catalog from site XML"); } } @@ -327,18 +570,17 @@ private static void initializeAuthToLocal() { * @return true if user is allowed, false otherwise */ private boolean isAllowedServiceUser(String serviceName, String userName) { - boolean ret; - - if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) { - Set allowedUsers = allowedServiceUsers.get(serviceName); - - ret = allowedUsers != null && allowedUsers.contains(userName); - } else { - ret = false; - } - - LOG.debug("isAllowedServiceUser(serviceName={}, userName={}): ret={}", serviceName, userName, ret); - + boolean dynamicEnabled = partitionPlanService != null + && partitionPlanService.isDynamicPartitionPlanEnabled(); + boolean ret = ServiceAllowlistResolver.isAllowedServiceUser( + serviceName, + userName, + dynamicEnabled, + PartitionPlanHolder.getInstance(), + allowedServiceUsers); + LOG.debug( + "isAllowedServiceUser(serviceName={}, userName={}, dynamic={}): ret={}", + serviceName, userName, dynamicEnabled, ret); return ret; } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java index 185ca100009..ba25d93419f 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServerConfig.java @@ -29,7 +29,8 @@ public class AuditServerConfig extends AuditConfig { private static final Logger LOG = LoggerFactory.getLogger(AuditServerConfig.class); - private static final String CONFIG_FILE_PATH = "conf/ranger-audit-ingestor-site.xml"; + private static final String CONFIG_FILE_PATH = "conf/ranger-audit-ingestor-site.xml"; + private static final String AUDIT_CONFIG_PROPERTY = "audit.config"; private static volatile AuditServerConfig sInstance; @@ -60,9 +61,10 @@ private boolean addAuditServerResources() { boolean ret = true; - // Load ranger-audit-ingestor-site.xml - if (!addAuditResource(CONFIG_FILE_PATH, true)) { - LOG.error("Could not load required configuration: {}", CONFIG_FILE_PATH); + // Prefer -Daudit.config (external conf dir) over classpath WEB-INF copy + String configPath = System.getProperty(AUDIT_CONFIG_PROPERTY, CONFIG_FILE_PATH); + if (!addAuditResource(configPath, true)) { + LOG.error("Could not load required configuration: {}", configPath); ret = false; } diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 1688b4dc43d..6adba2ca1e4 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -184,7 +184,7 @@ ranger.audit.ingestor.service.dev_hdfs.allowed.users - hdfs + hdfs,nn Allowed users for dev_hdfs (Policy Manager service name; from policy.download.auth.users) @@ -240,6 +240,7 @@ ranger.audit.ingestor.service.dev_solr.allowed.users solr Allowed users for dev_solr (Solr plugin) + @@ -249,6 +250,21 @@ RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ + RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ + RULE:[2:$1/$2@$0](hbase/.*@.*)s/.*/hbase/ + RULE:[2:$1/$2@$0](kafka/.*@.*)s/.*/kafka/ + RULE:[2:$1/$2@$0](knox/.*@.*)s/.*/knox/ + RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ + RULE:[2:$1/$2@$0](trino/.*@.*)s/.*/trino/ + RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ + RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ + RULE:[2:$1/$2@$0](scm/.*@.*)s/.*/scm/ + RULE:[2:$1/$2@$0](dn/.*@.*)s/.*/dn/ + RULE:[2:$1/$2@$0](solr/.*@.*)s/.*/solr/ + RULE:[2:$1/$2@$0](rangertagsync/.*@.*)s/.*/rangertagsync/ + RULE:[2:$1/$2@$0](atlas/.*@.*)s/.*/atlas/ + RULE:[2:$1/$2@$0](kudu/.*@.*)s/.*/kudu/ + RULE:[2:$1/$2@$0](nifi/.*@.*)s/.*/nifi/ RULE:[1:$1@$0](.*@.*)s/@.*// DEFAULT @@ -265,6 +281,21 @@ - RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ # nn,dn,jn,hdfs/* -> hdfs (dev_hdfs) - RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ # rm,nm,yarn/* -> yarn (dev_yarn) - RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ # jhs/* -> mapred + - RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ # hive/* -> hive (dev_hive) + - RULE:[2:$1/$2@$0](hbase/.*@.*)s/.*/hbase/ # hbase/* -> hbase (dev_hbase) + - RULE:[2:$1/$2@$0](kafka/.*@.*)s/.*/kafka/ # kafka/* -> kafka (dev_kafka) + - RULE:[2:$1/$2@$0](knox/.*@.*)s/.*/knox/ # knox/* -> knox (dev_knox) + - RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ # rangerkms/* -> rangerkms (dev_kms) + - RULE:[2:$1/$2@$0](trino/.*@.*)s/.*/trino/ # trino/* -> trino (dev_trino) + - RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ # ozone/* -> ozone (dev_ozone) + - RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ # om/* -> om (dev_ozone) + - RULE:[2:$1/$2@$0](scm/.*@.*)s/.*/scm/ # scm/* -> scm (dev_ozone) + - RULE:[2:$1/$2@$0](dn/.*@.*)s/.*/dn/ # dn/* -> dn (dev_ozone) + - RULE:[2:$1/$2@$0](solr/.*@.*)s/.*/solr/ # solr/* -> solr (dev_solr) + - RULE:[2:$1/$2@$0](rangertagsync/.*@.*)s/.*/rangertagsync/ # rangertagsync/* -> rangertagsync (dev_tag) + - RULE:[2:$1/$2@$0](atlas/.*@.*)s/.*/atlas/ # atlas/* -> atlas (dev_atlas) + - RULE:[2:$1/$2@$0](kudu/.*@.*)s/.*/kudu/ # kudu/* -> kudu (dev_kudu) + - RULE:[2:$1/$2@$0](nifi/.*@.*)s/.*/nifi/ # nifi/* -> nifi (dev_nifi) - RULE:[1:$1@$0](.*@.*)s/@.*// # user@REALM -> user - DEFAULT ]]> @@ -322,12 +353,27 @@ ranger.audit.ingestor.kafka.configured.plugins - hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi + - Comma-separated list of configured plugin IDs. - If set: Uses AuditPartitioner with auto-calculated partitions (sum of plugin allocations + buffer). - If empty/not set: Uses Kafka default hash-based partitioner with topic.partitions value. - Each plugin receives dedicated partitions based on topic.partitions.per.configured.plugin (default: 3). + Comma-separated plugin IDs (agentId / Kafka record key) that receive dedicated partition ranges in static mode. + Leave empty by default; add only the plugins you deploy. + + Empty (recommended greenfield): + - Static mode: Kafka default hash partitioner using kafka.topic.partitions below. + - Dynamic mode (kafka.partition.plan.dynamic.enabled=true): first bootstrap plan uses all + kafka.topic.partitions as buffer; onboard plugins via POST /api/audit/partition-plan/services + or promote without editing this list. + + Non-empty (static / bootstrap seed): + - Uses AuditPartitioner: total partitions = sum(per-plugin counts) + kafka.topic.partitions.buffer. + - Append new plugin IDs only; never reorder existing entries (partition ranges are contiguous). + - Dynamic mode: seeds the initial plan from this list when ranger_audit_partition_plan is empty; + runtime changes use partition-plan REST, not XML edits. + + Example plugin IDs (pick what you run): hdfs, yarn, knox, hiveServer2, hiveMetastore, kafka, + hbaseRegional, hbaseMaster, solr, trino, ozone, kudu, nifi. + Per-plugin partition count: kafka.topic.partitions.per.configured.plugin (default 3), overridable via + kafka.plugin.partition.overrides.{pluginId}. @@ -349,8 +395,8 @@ Number of buffer partitions reserved for unconfigured plugins. Used ONLY when configured.plugins is set (for plugin-based partitioning). Total = (sum of plugin partitions) + (buffer partitions). - Example: 7 plugins × 3 + 9 buffer = 30 total. - Ignored when configured.plugins is empty. + Example: 2 plugins × 3 + 9 buffer = 15 total. + Ignored when configured.plugins is empty (hash-based or dynamic buffer-only bootstrap). @@ -401,6 +447,45 @@ + + + ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled + false + + false or absent = legacy XML AuditPartitioner at startup (today's behavior). + true = Kafka registry + REST + PartitionPlanWatcher. + + + + ranger.audit.ingestor.kafka.security.protocol SASL_PLAINTEXT diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java new file mode 100644 index 00000000000..7cad59f43a7 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka; + +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrap; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrapConfig; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AuditPartitionerDynamicTest { + private static final String TOPIC = "ranger_audits"; + private static final Node BROKER = new Node(0, "localhost", 9092); + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testStaticModeUnchangedWhenDynamicDisabled() { + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(staticProducerConfig()); + + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(1, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(2, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertTrue(partitioner.partition(TOPIC, "unknownPlugin", null, null, null, cluster(15)) >= 6); + } + + @Test + public void testDynamicModeRoutesConfiguredPluginRoundRobin() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + PartitionPlanHolder.getInstance().install(plan, 15); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(1, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(2, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertEquals(0, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(15))); + assertIterableEquals(List.of(3, 4, 5), List.of( + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)), + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)), + partitioner.partition(TOPIC, "hiveServer2", null, null, null, cluster(15)))); + } + + @Test + public void testDynamicModeRoutesUnknownPluginToBuffer() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 3)); + PartitionPlanHolder.getInstance().install(plan, 6); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + int partition = partitioner.partition(TOPIC, "trino", null, null, null, cluster(6)); + assertTrue(partition >= 3 && partition <= 5); + } + + @Test + public void testDynamicModeUsesPlannedTailPartitionWhenClusterMetadataLagsAfterScale() { + PartitionPlan planV1 = PartitionPlanBootstrap.createInitialPlan( + PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 9)); + PartitionPlanHolder.getInstance().install(planV1, 12); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + + // Simulate scale: plan now assigns tail partitions 12,13 but producer metadata still shows 12 partitions. + PartitionPlan planV2 = planV1.toBuilder() + .version(2) + .topicPartitionCount(14) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2, 12, 13))) + .buffer(PluginPartitionAssignment.ofRange(3, 11)) + .updatedBy("test") + .build(); + PartitionPlanHolder.getInstance().install(planV2, 14); + + // Round-robin index 3 -> planned partition 12; must not clamp to 11 when cluster still reports 12 partitions. + for (int i = 0; i < 3; i++) { + partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(12)); + } + assertEquals(12, partitioner.partition(TOPIC, "hdfs", null, null, null, cluster(12))); + } + + @Test + public void testConcurrentPartitionWhilePlanSwaps() throws Exception { + PartitionPlan planV1 = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 2, 2)); + PartitionPlanHolder.getInstance().install(planV1, 4); + + AuditPartitioner partitioner = new AuditPartitioner(); + partitioner.configure(dynamicProducerConfig()); + Cluster cluster = cluster(4); + + PartitionPlan planV2 = planV1.toBuilder().version(2).updatedBy("test").build(); + + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch startLatch = new CountDownLatch(1); + AtomicInteger errors = new AtomicInteger(); + Set seen = Collections.synchronizedSet(new HashSet<>()); + + for (int i = 0; i < 8; i++) { + executor.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < 200; j++) { + int p = partitioner.partition(TOPIC, "hdfs", null, null, null, cluster); + seen.add(p); + } + } catch (Exception e) { + errors.incrementAndGet(); + } + }); + } + + startLatch.countDown(); + PartitionPlanHolder.getInstance().install(planV2, 4); + executor.shutdown(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + assertEquals(0, errors.get()); + assertTrue(seen.stream().allMatch(p -> p >= 0 && p < 4)); + } + + private static Map staticProducerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map config = new HashMap<>(); + config.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS, 15); + config.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + return config; + } + + private static Map dynamicProducerConfig() { + Map config = staticProducerConfig(); + config.put(AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.substring(0, AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.length() - 1) + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return config; + } + + private static Cluster cluster(int partitionCount) { + PartitionInfo[] partitionInfos = new PartitionInfo[partitionCount]; + for (int i = 0; i < partitionCount; i++) { + partitionInfos[i] = new PartitionInfo(TOPIC, i, BROKER, new Node[] {BROKER}, new Node[] {BROKER}); + } + return new Cluster("cluster", Collections.singletonList(BROKER), List.of(partitionInfos), Collections.emptySet(), Collections.emptySet()); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java new file mode 100644 index 00000000000..91c2323416b --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AuthToLocalRuleComposerTest { + private static final String SAMPLE_CATALOG = + "RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/ " + + "RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/ " + + "RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/ " + + "RULE:[2:$1/$2@$0](hive/.*@.*)s/.*/hive/ " + + "RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/ " + + "RULE:[2:$1/$2@$0](om/.*@.*)s/.*/om/ " + + "RULE:[2:$1/$2@$0](ozone/.*@.*)s/.*/ozone/ " + + "RULE:[1:$1@$0](.*@.*)s/@.*// " + + "DEFAULT"; + + private static final String SENTINEL_NN_RULE = + "RULE:[2:$1/$2@$0](nn/.*@.*)s/.*/startup-not-applied/ DEFAULT"; + + private AuthToLocalRuleComposer composer; + + @BeforeEach + public void setUp() { + composer = AuthToLocalRuleComposer.getInstance(); + composer.resetForTests(); + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + composer.initializeFromProperties(props); + } + + @AfterEach + public void tearDown() { + composer.resetForTests(); + PartitionPlanHolder.getInstance().resetForTests(); + AuditServerConfig.getInstance().set( + PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, + "false"); + } + + @Test + public void testComposeKmsOnlyIncludesRangerkmsRuleAndTail() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("rangerkms")); + + assertTrue(rules.contains("(rangerkms/.*@.*)s/.*/rangerkms/")); + assertFalse(rules.contains("(hive/.*@.*)s/.*/hive/")); + assertTrue(rules.contains("RULE:[1:$1@$0](.*@.*)s/@.*//")); + assertTrue(rules.endsWith("DEFAULT")); + } + + @Test + public void testComposeHdfsUsesCatalogRegexNotSimpleRule() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("hdfs")); + + assertTrue(rules.contains("([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/")); + assertFalse(rules.contains("(hdfs/.*@.*)s/.*/hdfs/")); + } + + @Test + public void testComposeOzoneIncludesMultipleShortNames() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("ozone", "om", "scm", "dn")); + + assertTrue(rules.contains("(ozone/.*@.*)s/.*/ozone/")); + assertTrue(rules.contains("(om/.*@.*)s/.*/om/")); + } + + @Test + public void testComposeUnknownShortNameGeneratesSimpleRule() { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("myapp")); + + assertTrue(rules.contains("(myapp/.*@.*)s/.*/myapp/")); + } + + @Test + public void testCollectAllowedUserShortNamesFromPlan() { + Map services = new LinkedHashMap<>(); + services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); + services.put("dev_ozone", ServiceAllowlistEntry.ofUsers("om", "ozone")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("kms", PluginPartitionAssignment.of(0, 1))) + .buffer(PluginPartitionAssignment.of(2, 3, 4, 5)) + .services(services) + .build(); + + Set activeShortNames = AuthToLocalRuleComposer.collectAllowedUserShortNames(plan); + + assertEquals(Set.of("rangerkms", "om", "ozone"), activeShortNames); + } + + @Test + public void testKerberosMappingForComposedKmsRules() throws Exception { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("rangerkms")); + KerberosName.setRules(rules); + + assertEquals("rangerkms", new KerberosName("rangerkms/ranger-kms.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testKerberosMappingForComposedHdfsRules() throws Exception { + String rules = composer.composeKerberosRulesForAllowedShortNames(Set.of("hdfs")); + KerberosName.setRules(rules); + + assertEquals("hdfs", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenTopicMissingAppliesStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPartitionPlanTopicExistsTestOverride(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("hdfs", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenTopicExistsDefersStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPartitionPlanTopicExistsTestOverride(true); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testStartupDynamicWhenDisabledIsNoOp() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "false"); + composer.setPartitionPlanTopicExistsTestOverride(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testInstallPlanAppliesComposedAuthToLocalRules() throws Exception { + AuditServerConfig config = AuditServerConfig.getInstance(); + config.set(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + config.set(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + composer.initializeFromConfig(); + + Map services = new LinkedHashMap<>(); + services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(3) + .topicPartitionCount(6) + .plugins(Map.of("kms", PluginPartitionAssignment.of(0, 1))) + .buffer(PluginPartitionAssignment.of(2, 3, 4, 5)) + .services(services) + .build(); + + PartitionPlanHolder.getInstance().install(plan, 6); + + assertEquals("rangerkms", new KerberosName("rangerkms/ranger-kms.rangernw@EXAMPLE.COM").getShortName()); + assertThrows(KerberosName.NoMatchingRule.class, + () -> new KerberosName("nn/namenode.rangernw@EXAMPLE.COM").getShortName()); + } + + @Test + public void testExtractMappedShortNameFromCatalogLines() { + assertEquals("hdfs", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/")); + assertEquals("rangerkms", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0](rangerkms/.*@.*)s/.*/rangerkms/")); + assertEquals("mapred", AuthToLocalRuleCatalog.extractMappedShortName( + "RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/")); + } + + private static Properties dynamicModeProps() { + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java new file mode 100644 index 00000000000..0c416ac6084 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanAllocatorTest { + private PartitionPlan initialPlan; + + @BeforeEach + public void setUp() { + initialPlan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + @Test + public void testPromotePluginFromBuffer() { + PartitionPlan next = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + + assertEquals(2, next.getVersion()); + assertEquals(15, next.getTopicPartitionCount()); + assertIterableEquals(List.of(6, 7, 8), next.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of(9, 10, 11, 12, 13, 14), next.getBuffer().getPartitions()); + assertIterableEquals(List.of(0, 1, 2), next.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(3, 4, 5), next.getPlugins().get("hiveServer2").getPartitions()); + } + + @Test + public void testPromotePluginGrowsTopicWhenBufferInsufficient() { + PartitionPlan next = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 12, "ops"); + + assertEquals(18, next.getTopicPartitionCount()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17), next.getPlugins().get("trino").getPartitions()); + assertEquals(0, next.getBuffer().size()); + } + + @Test + public void testScalePluginAppendsTailOnly() { + PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + PartitionPlan scaled = PartitionPlanAllocator.scalePlugin(promoted, "hiveServer2", 3, "ops"); + + assertEquals(3, scaled.getVersion()); + assertEquals(18, scaled.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17), scaled.getPlugins().get("hiveServer2").getPartitions()); + assertIterableEquals(List.of(0, 1, 2), scaled.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8), scaled.getPlugins().get("trino").getPartitions()); + } + + @Test + public void testPromoteAlreadyConfiguredPluginFails() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.promotePlugin(initialPlan, "hdfs", 1, "ops")); + assertTrue(error.getMessage().contains("requested 1")); + } + + @Test + public void testIsPromoteAlreadyAppliedWhenPluginAndCountMatch() { + PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + + assertTrue(PartitionPlanAllocator.isPromoteAlreadyApplied(promoted, "trino", 3, null, null)); + assertFalse(PartitionPlanAllocator.isPromoteAlreadyApplied(promoted, "trino", 5, null, null)); + } + + @Test + public void testIsOnboardAlreadyAppliedWhenAllowlistAndPluginMatch() { + PartitionPlan onboarded = PartitionPlanAllocator.onboardRepo(initialPlan, "dev_trino", "trino", 3, List.of("trino"), "ops"); + + assertTrue(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "dev_trino", "trino", 3, List.of("trino"))); + assertFalse(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "dev_trino", "trino", 3, List.of("other"))); + } + + @Test + public void testPromoteConflictWhenPartitionCountDiffers() { + PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.promotePlugin(promoted, "trino", 5, "ops")); + + assertTrue(error.getMessage().contains("requested 5")); + } + + @Test + public void testScaleUnknownPluginFails() { + assertThrows(PartitionPlanException.class, () -> PartitionPlanAllocator.scalePlugin(initialPlan, "trino", 2, "ops")); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java new file mode 100644 index 00000000000..3a3878e5534 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapSupportTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +public class PartitionPlanBootstrapSupportTest { + private static final String TOPIC = "ranger_audits"; + + @Test + public void testBootstrapReturnsExistingPlan() { + PartitionPlan existing = samplePlan(); + InMemoryPartitionPlanRegistry registry = new InMemoryPartitionPlanRegistry(existing); + Map config = producerConfig(); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, config); + + assertEquals(existing, plan); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testBootstrapPublishesV1WhenEmpty() { + InMemoryPartitionPlanRegistry registry = new InMemoryPartitionPlanRegistry(null); + Map config = producerConfig(); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, config); + + assertEquals(1, plan.getVersion()); + assertEquals(15, plan.getTopicPartitionCount()); + assertIterableEquals(plan.getPlugins().get("hdfs").getPartitions(), registry.readPlan(TOPIC).getPlugins().get("hdfs").getPartitions()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testBootstrapAdoptsPeerPlanBeforePublish() { + PartitionPlan peerPlan = samplePlan().toBuilder().updatedBy("peer").build(); + PeerPublishesOnSecondReadRegistry registry = new PeerPublishesOnSecondReadRegistry(peerPlan); + + PartitionPlan plan = PartitionPlanBootstrap.bootstrapIfEmpty(registry, TOPIC, producerConfig()); + + assertEquals(peerPlan, plan); + assertEquals(0, registry.getWriteCount()); + } + + private static Map producerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map config = new HashMap<>(); + config.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + config.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + config.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + return config; + } + + private static PartitionPlan samplePlan() { + return PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + private static final class PeerPublishesOnSecondReadRegistry implements PartitionPlanRegistry { + private final PartitionPlan peerPlan; + private int readCount; + private int writeCount; + + private PeerPublishesOnSecondReadRegistry(PartitionPlan peerPlan) { + this.peerPlan = peerPlan; + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + readCount++; + if (readCount >= 2) { + return peerPlan; + } + return null; + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + writeCount++; + } + + @Override + public void close() { + } + + private int getWriteCount() { + return writeCount; + } + } + + private static class InMemoryPartitionPlanRegistry implements PartitionPlanRegistry { + private final AtomicReference planRef; + private int writeCount; + + private InMemoryPartitionPlanRegistry(PartitionPlan initialPlan) { + this.planRef = new AtomicReference<>(initialPlan); + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + return planRef.get(); + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan plan) { + planRef.set(plan); + writeCount++; + } + + @Override + public void close() { + } + + private int getWriteCount() { + return writeCount; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java new file mode 100644 index 00000000000..84c1a3ceef7 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +public class PartitionPlanBootstrapTest { + private static final String TOPIC = "ranger_audits"; + + @Test + public void testCreateInitialPlanMatchesStaticPartitionerLayout() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + + assertEquals(TOPIC, plan.getTopic()); + assertEquals(1, plan.getVersion()); + assertEquals(15, plan.getTopicPartitionCount()); + assertEquals(PartitionPlanConstants.BOOTSTRAP_UPDATED_BY, plan.getUpdatedBy()); + assertIterableEquals(List.of(0, 1, 2), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(3, 4, 5), plan.getPlugins().get("hiveServer2").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanHonorsPluginOverrides() { + PartitionPlanBootstrapConfig config = PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "trino"}, 3, 4).withPluginOverride("hdfs", 5); + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(config); + + assertEquals(12, plan.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2, 3, 4), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(5, 6, 7), plan.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of(8, 9, 10, 11), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanEmptyPluginsUsesHashBasedTopicPartitions() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan( + new PartitionPlanBootstrapConfig(TOPIC, new String[0], 3, 9, 10, Collections.emptyMap())); + + assertEquals(10, plan.getTopicPartitionCount()); + assertEquals(0, plan.getPlugins().size()); + assertIterableEquals(List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), plan.getBuffer().getPartitions()); + } + + @Test + public void testCreateInitialPlanFromProducerConfig() { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + Map configs = new HashMap<>(); + configs.put(propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, "hdfs,hiveServer2"); + configs.put(propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, 3); + configs.put(propPrefix + AuditServerConstants.PROP_BUFFER_PARTITIONS, 9); + + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlanFromProducerConfig(configs, TOPIC); + + assertEquals(15, plan.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2), plan.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), plan.getBuffer().getPartitions()); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java new file mode 100644 index 00000000000..9aa4b0de80a --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanHolderTest { + private static final int TOPIC_PARTITIONS = 6; + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testGetAllowedUsersReturnsNullWhenNoPlanInstalled() { + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + } + + @Test + public void testGetAllowedUsersReturnsNullWhenServicesBlockMissing() { + PartitionPlan plan = basePlanBuilder() + .build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + } + + @Test + public void testGetAllowedUsersReturnsNullForRepoNotInPartialPlan() { + Map services = new LinkedHashMap<>(); + services.put("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs", "nn")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + assertIterableEquals(List.of("hdfs", "nn"), PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hdfs")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + assertNull(PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_trino")); + } + + @Test + public void testGetAllowedUsersReturnsRegistryUsersForOnboardedRepo() { + Map services = Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS); + + Set allowed = PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive"); + + assertIterableEquals(List.of("hive"), allowed); + assertTrue(allowed != null && !allowed.contains("hdfs")); + } + + @Test + public void testInstallRejectsEmptyServiceAllowlist() { + Map services = Map.of("dev_hdfs", new ServiceAllowlistEntry(List.of(), "test", null)); + PartitionPlan plan = basePlanBuilder().services(services).build(); + + assertThrows(PartitionPlanException.class, () -> PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS)); + } + + private static PartitionPlan.Builder basePlanBuilder() { + return PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(TOPIC_PARTITIONS) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java new file mode 100644 index 00000000000..5b813f0dc3d --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfigTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanKafkaConfigTest { + private static final String PROP_PREFIX = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER + "kafka"; + + @Test + public void testResolvePlanTopicNameUsesDefault() { + Properties props = new Properties(); + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC, PartitionPlanKafkaConfig.resolvePlanTopicName(props, PROP_PREFIX)); + } + + @Test + public void testResolvePlanTopicNameUsesOverride() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, "custom_plan_topic"); + assertEquals("custom_plan_topic", PartitionPlanKafkaConfig.resolvePlanTopicName(props, PROP_PREFIX)); + } + + @Test + public void testDynamicPartitionPlanDisabledByDefault() { + assertFalse(PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(new Properties(), PROP_PREFIX)); + } + + @Test + public void testDynamicPartitionPlanEnabledFromProperty() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + assertTrue(PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(props, PROP_PREFIX)); + } + + @Test + public void testResolveRefreshIntervalUsesDefault() { + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_REFRESH_INTERVAL_MS, PartitionPlanKafkaConfig.resolveRefreshIntervalMs(new Properties(), PROP_PREFIX)); + } + + @Test + public void testResolveConsumerPollTimeoutUsesDefault() { + assertEquals(AuditServerConstants.DEFAULT_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(new Properties(), PROP_PREFIX)); + } + + @Test + public void testResolveConsumerPollTimeoutUsesOverride() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_CONSUMER_POLL_TIMEOUT_MS, "1000"); + assertEquals(1000, PartitionPlanKafkaConfig.resolveConsumerPollTimeoutMs(props, PROP_PREFIX)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java new file mode 100644 index 00000000000..fb294867fe9 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanReplacementTest { + private static final PartitionPlan CURRENT = PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(9) + .plugins(Map.of( + "hdfs", PluginPartitionAssignment.of(0, 1, 2), + "hiveServer2", PluginPartitionAssignment.of(3, 4, 5))) + .buffer(PluginPartitionAssignment.of(6, 7, 8)) + .services(Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs"))) + .build(); + + @Test + public void testMergeInheritsEmptyOptionalFields() { + PartitionPlanReplacement update = new PartitionPlanReplacement(1, null, null, null, null); + + PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); + + assertEquals(9, merged.getTopicPartitionCount()); + assertEquals(CURRENT.getPlugins(), merged.getPlugins()); + assertEquals(CURRENT.getBuffer(), merged.getBuffer()); + assertEquals(CURRENT.getServices(), merged.getServices()); + } + + @Test + public void testMergeAddsOnlyNewPluginAndServiceEntries() { + PartitionPlanReplacement update = new PartitionPlanReplacement( + 1, + null, + Map.of("trino", PluginPartitionAssignment.of(6, 7, 8)), + null, + Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino"))); + + PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); + + assertTrue(merged.getPlugins().containsKey("trino")); + assertTrue(merged.getServices().containsKey("dev_trino")); + assertTrue(merged.getPlugins().containsKey("hdfs")); + assertTrue(merged.getServices().containsKey("dev_hdfs")); + } + + @Test + public void testHasMergeDeltaRequiresNonEmptyField() { + assertFalse(new PartitionPlanReplacement(1, null, null, null, null).hasMergeDelta()); + assertTrue(new PartitionPlanReplacement(1, 12, null, null, null).hasMergeDelta()); + assertTrue(new PartitionPlanReplacement(1, null, Map.of("trino", PluginPartitionAssignment.of(6)), null, null).hasMergeDelta()); + } + + @Test + public void testMergeUpsertsExistingServiceAllowlist() { + PartitionPlanReplacement update = new PartitionPlanReplacement( + 1, + null, + null, + null, + Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs", "backup"))); + + PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); + + assertIterableEquals(List.of("hdfs", "backup"), merged.getServices().get("dev_hdfs").getAllowedUsers()); + } + + @Test + public void testMergeExistingPluginWithSameAssignmentIsNoOpDelta() { + PartitionPlanReplacement update = new PartitionPlanReplacement( + 1, + null, + Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2)), + null, + null); + + PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); + + assertTrue(CURRENT.sameContentAs(merged)); + } + + @Test + public void testMergeExistingPluginWithDifferentAssignmentFails() { + PartitionPlanReplacement update = new PartitionPlanReplacement( + 1, + null, + Map.of("hdfs", PluginPartitionAssignment.of(0, 1)), + null, + null); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, () -> update.toMergedPlan(CURRENT, "ops")); + + assertTrue(error.getMessage().contains("different partition assignment")); + } + + @Test + public void testSameContentAsIgnoresVersionAndMetadata() { + PartitionPlan withMetadata = CURRENT.toBuilder().version(99).updatedAt("later").updatedBy("other").build(); + + assertTrue(CURRENT.sameContentAs(withMetadata)); + } + + @Test + public void testRequestValidatorRejectsBlankPromotePluginId() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validatePromotePlugin(new PromotePlugin("", 2, 1))); + } + + @Test + public void testRequestValidatorRejectsOnboardServiceWithoutAllowedUsers() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardService( + new OnboardService("dev_trino", "trino", 2, List.of(" "), 1))); + } + + @Test + public void testRequestValidatorRejectsScaleWithZeroAdditionalPartitions() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateScalePlugin("hiveServer2", new PluginScale(0, 1))); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java new file mode 100644 index 00000000000..ce2ad8b56e7 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanServiceMutationTest { + private static final String TOPIC = "ranger_audits"; + + private PartitionPlan initialPlan; + + @BeforeEach + public void setUp() { + initialPlan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs", "hiveServer2"}, 3, 9)); + } + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testPromotePluginPublishesNextVersion() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(2, registry.getPlan().getVersion()); + assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); + assertEquals(1, registry.getWriteCount()); + assertEquals(result, PartitionPlanHolder.getInstance().getPlan()); + } + + @Test + public void testMergePlanAddsNewPluginOnly() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + PartitionPlanReplacement request = new PartitionPlanReplacement( + 1, + null, + Map.of("trino", PluginPartitionAssignment.of(6, 7, 8)), + null, + null); + + PartitionPlan result = service.mergePartitionPlan(request, "ops"); + + assertEquals(2, result.getVersion()); + assertIterableEquals(List.of(0, 1, 2), result.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(3, 4, 5), result.getPlugins().get("hiveServer2").getPartitions()); + assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of(9, 10, 11, 12, 13, 14), result.getBuffer().getPartitions()); + } + + @Test + public void testScalePluginAppendsTailPartitionsViaDedicatedEndpoint() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.scalePlugin("hiveServer2", new PluginScale(2, 1), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(17, result.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16), result.getPlugins().get("hiveServer2").getPartitions()); + } + + @Test + public void testMergePlanRejectsExistingPluginDelta() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + PartitionPlanReplacement request = new PartitionPlanReplacement( + 1, + null, + Map.of("hiveServer2", PluginPartitionAssignment.of(3, 4, 5, 15, 16)), + null, + null); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.mergePartitionPlan(request, "ops")); + + assertTrue(error.getMessage().contains("different partition assignment")); + } + + @Test + public void testMergePlanCanAddServices() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_new_hive", ServiceAllowlistEntry.ofUsers("hive")); + PartitionPlanReplacement request = new PartitionPlanReplacement(1, null, null, null, services); + + PartitionPlan result = service.mergePartitionPlan(request, "ops"); + + assertEquals(2, result.getVersion()); + assertIterableEquals(List.of("hive"), result.getServices().get("dev_new_hive").getAllowedUsers()); + } + + @Test + public void testPromotePluginRejectsBlankPluginId() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.promotePlugin(new PromotePlugin(" ", 3, 1), "ops")); + + assertTrue(error.getMessage().contains("pluginId")); + } + + @Test + public void testScalePluginRejectsInvalidAdditionalPartitions() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.scalePlugin("hiveServer2", new PluginScale(0, 1), "ops")); + + assertTrue(error.getMessage().contains("additionalPartitions")); + } + + @Test + public void testOnboardServiceRejectsMissingAllowedUsers() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardService(new OnboardService("dev_trino", "trino", 3, List.of(), 1), "ops")); + + assertTrue(error.getMessage().contains("allowedUsers")); + } + + @Test + public void testOnboardRepoPromotesPluginAndUpsertsAllowlist() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.onboardService(new OnboardService("dev_trino", "trino", 3, List.of("trino"), 1), "ops"); + + assertEquals(2, result.getVersion()); + assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); + assertIterableEquals(List.of("trino"), result.getServices().get("dev_trino").getAllowedUsers()); + } + + @Test + public void testPromoteWithAllowlistUpsertsServices() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlan result = service.promotePlugin(new PromotePlugin("trino", 3, 1, "dev_trino", List.of("trino")), "ops"); + + assertEquals(2, result.getVersion()); + assertIterableEquals(List.of("trino"), result.getServices().get("dev_trino").getAllowedUsers()); + } + + @Test + public void testStaleExpectedVersionReturnsConflict() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, + () -> service.promotePlugin(new PromotePlugin("trino", 3, 99), "ops")); + + assertEquals(initialPlan, conflict.getCurrentPlan()); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testConflictWhenPeerPublishedBeforeWrite() { + PartitionPlan peerPlan = initialPlan.toBuilder().version(2).updatedBy("peer").build(); + MutableRegistry registry = new MutableRegistry(initialPlan) { + private final AtomicInteger reads = new AtomicInteger(); + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + if (reads.incrementAndGet() >= 2) { + return peerPlan; + } + return super.readPlan(auditTopicKey); + } + }; + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, + () -> service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops")); + + assertEquals(peerPlan, conflict.getCurrentPlan()); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testTopicGrowFailureSurfacesAsPlanException() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new FailingAuditTopicPartitionGrower()); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.promotePlugin(new PromotePlugin("trino", 12, 1), "ops")); + + assertTrue(error.getMessage().contains("grow audit topic")); + assertEquals(0, registry.getWriteCount()); + } + + @Test + public void testDuplicateOnboardReturnsCurrentPlanWithoutWrite() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + OnboardService request = new OnboardService("dev_trino", "trino", 3, List.of("trino"), 1); + + PartitionPlan first = service.onboardService(request, "ops"); + assertEquals(1, registry.getWriteCount()); + + OnboardService retry = new OnboardService("dev_trino", "trino", 3, List.of("trino"), first.getVersion()); + PartitionPlan second = service.onboardService(retry, "ops"); + + assertEquals(first, second); + assertEquals(2, second.getVersion()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testDuplicatePromoteReturnsCurrentPlanWithoutWrite() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + PromotePlugin request = new PromotePlugin("trino", 3, 1); + + PartitionPlan first = service.promotePlugin(request, "ops"); + assertEquals(1, registry.getWriteCount()); + + PromotePlugin retry = new PromotePlugin("trino", 3, first.getVersion()); + PartitionPlan second = service.promotePlugin(retry, "ops"); + + assertEquals(first, second); + assertEquals(2, second.getVersion()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testPromoteConflictWhenPartitionCountDiffers() { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops"); + + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.promotePlugin(new PromotePlugin("trino", 5, 2), "ops")); + + assertTrue(error.getMessage().contains("requested 5")); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testMergePlanNoOpWhenServicesUnchanged() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + service.mergePartitionPlan(new PartitionPlanReplacement(1, null, null, null, services), "ops"); + assertEquals(1, registry.getWriteCount()); + + PartitionPlan result = service.mergePartitionPlan(new PartitionPlanReplacement(2, null, null, null, services), "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testMergePlanStillAppliesWhenServicesChange() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_new_hive", ServiceAllowlistEntry.ofUsers("hive")); + PartitionPlanReplacement request = new PartitionPlanReplacement(1, null, null, null, services); + + PartitionPlan result = service.mergePartitionPlan(request, "ops"); + + assertEquals(2, result.getVersion()); + assertEquals(1, registry.getWriteCount()); + } + + @Test + public void testScalePluginStillAppendsOnRepeat() throws Exception { + MutableRegistry registry = new MutableRegistry(initialPlan); + PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + + service.scalePlugin("hiveServer2", new PluginScale(2, 1), "ops"); + assertEquals(1, registry.getWriteCount()); + + PartitionPlan second = service.scalePlugin("hiveServer2", new PluginScale(2, 2), "ops"); + + assertEquals(3, second.getVersion()); + assertEquals(2, registry.getWriteCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17, 18), second.getPlugins().get("hiveServer2").getPartitions()); + } + + private static PartitionPlanService service(MutableRegistry registry, KafkaAuditTopicPartitionGrower topicGrower) { + return new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new FixedPartitionPlanRegistryFactory(registry), topicGrower); + } + + private static Properties enabledConfig() { + Properties props = new Properties(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, TOPIC); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } + + private static final class FixedPartitionPlanRegistryFactory extends PartitionPlanRegistryFactory { + private final PartitionPlanRegistry registry; + + private FixedPartitionPlanRegistryFactory(PartitionPlanRegistry registry) { + this.registry = registry; + } + + @Override + public PartitionPlanRegistry open(Properties props, String propPrefix) { + return registry; + } + } + + private static final class NoOpAuditTopicPartitionGrower extends KafkaAuditTopicPartitionGrower { + @Override + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + } + } + + private static final class FailingAuditTopicPartitionGrower extends KafkaAuditTopicPartitionGrower { + @Override + public void growAuditTopicToRequiredPartitionCount(Properties props, String propPrefix, String auditTopicName, int requiredPartitionCount) { + throw new RuntimeException("kafka down"); + } + } + + private static class MutableRegistry implements PartitionPlanRegistry { + private PartitionPlan plan; + private int writeCount; + + private MutableRegistry(PartitionPlan plan) { + this.plan = plan; + } + + @Override + public PartitionPlan readPlan(String auditTopicKey) { + return plan; + } + + @Override + public void writePlan(String auditTopicKey, PartitionPlan newPlan) { + plan = newPlan; + writeCount++; + } + + @Override + public void close() { + } + + private PartitionPlan getPlan() { + return plan; + } + + private int getWriteCount() { + return writeCount; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java new file mode 100644 index 00000000000..254c597b921 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.server.AuditServerConstants; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanServiceTest { + private static final String TOPIC = "ranger_audits"; + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testDynamicDisabledWhenFlagOff() { + PartitionPlanService service = new PartitionPlanService(disabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertFalse(service.isDynamicPartitionPlanEnabled()); + } + + @Test + public void testDynamicEnabledWhenFlagOn() { + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertTrue(service.isDynamicPartitionPlanEnabled()); + } + + @Test + public void testGetFromMemoryReturnsInstalledPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create(TOPIC, new String[] {"hdfs"}, 3, 3)); + PartitionPlanHolder.getInstance().install(plan, 6); + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + + PartitionPlan loaded = service.getPartitionPlan(); + + assertEquals(plan, loaded); + assertEquals(plan.toJson(), service.getPartitionPlan().toJson()); + } + + @Test + public void testGetFromMemoryFailsWhenPlanNotLoaded() { + PartitionPlanService service = new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new NoOpPartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); + assertThrows(PartitionPlanException.class, () -> service.getPartitionPlan()); + } + + private static Properties disabledConfig() { + Properties props = baseConfig(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "false"); + return props; + } + + private static Properties enabledConfig() { + Properties props = baseConfig(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } + + private static Properties baseConfig() { + Properties props = new Properties(); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_TOPIC_NAME, TOPIC); + return props; + } + + private static final class NoOpPartitionPlanRegistryFactory extends PartitionPlanRegistryFactory { + @Override + public PartitionPlanRegistry open(Properties props, String propPrefix) { + return null; + } + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java new file mode 100644 index 00000000000..09b264ce263 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanUpdateApplierTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class PartitionPlanUpdateApplierTest { + private static final String AUDIT_TOPIC = "ranger_audits"; + private static final String PLAN_TOPIC = "ranger_audit_partition_plan"; + private static final int KAFKA_PARTITIONS = 6; + + private PartitionPlanHolder holder; + private PartitionPlanUpdateApplier applier; + + @BeforeEach + public void setUp() { + holder = PartitionPlanHolder.getInstance(); + holder.resetForTests(); + holder.install(planWithVersion(1), KAFKA_PARTITIONS); + applier = new PartitionPlanUpdateApplier(new Properties(), AUDIT_TOPIC, holder, () -> KAFKA_PARTITIONS); + } + + @AfterEach + public void tearDown() { + holder.resetForTests(); + } + + @Test + public void testIgnoresWrongRecordKey() { + applier.applyRecordIfNewer(record("other-topic", planWithVersion(2).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testIgnoresSameVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(1).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testIgnoresOlderVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(0).toJson())); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testInstallsNewerVersion() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(2).toJson())); + + assertEquals(2, holder.getLastInstalledVersion()); + assertEquals(2, holder.getPlan().getVersion()); + } + + @Test + public void testIgnoresInvalidJson() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, "{not-json")); + + assertEquals(1, holder.getLastInstalledVersion()); + } + + @Test + public void testStartupSyncCanCatchPlanPublishedBeforeWatcherThread() { + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(2).toJson())); + applier.applyRecordIfNewer(record(AUDIT_TOPIC, planWithVersion(3).toJson())); + + assertEquals(3, holder.getLastInstalledVersion()); + } + + private static PartitionPlan planWithVersion(int version) { + return PartitionPlan.builder() + .topic(AUDIT_TOPIC) + .version(version) + .topicPartitionCount(KAFKA_PARTITIONS) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .build(); + } + + private static ConsumerRecord record(String key, String value) { + return new ConsumerRecord<>(PLAN_TOPIC, 0, 0L, key, value); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java new file mode 100644 index 00000000000..83f732b885f --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanValidatorTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class PartitionPlanValidatorTest { + @Test + public void testValidateAcceptsWellFormedPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs"}, 2, 2)); + assertDoesNotThrow(() -> PartitionPlanValidator.validate(plan, 4)); + } + + @Test + public void testValidateRejectsDuplicatePartitionIds() { + Map plugins = new LinkedHashMap<>(); + plugins.put("hdfs", PluginPartitionAssignment.of(0, 1)); + PartitionPlan plan = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(3).plugins(plugins).buffer(PluginPartitionAssignment.of(1, 2)).build(); + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validate(plan)); + } + + @Test + public void testValidateRejectsKafkaPartitionMismatch() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs"}, 2, 2)); + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validate(plan, 10)); + } + + @Test + public void testValidateAppendOnlyRejectsReshuffle() { + PartitionPlan current = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(6).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5)).buffer(PluginPartitionAssignment.empty()).build(); + + Map reshuffled = new LinkedHashMap<>(); + reshuffled.put("hdfs", PluginPartitionAssignment.of(0, 1, 2, 3)); + reshuffled.put("hiveServer2", PluginPartitionAssignment.of(4, 5)); + PartitionPlan proposed = PartitionPlan.builder().topic("ranger_audits").version(2).topicPartitionCount(6).plugins(reshuffled).buffer(PluginPartitionAssignment.empty()).build(); + + assertThrows(PartitionPlanException.class, () -> PartitionPlanValidator.validateAppendOnly(current, proposed)); + } + + @Test + public void testValidateAppendOnlyAcceptsTailGrowth() { + PartitionPlan current = PartitionPlan.builder().topic("ranger_audits").version(1).topicPartitionCount(6).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5)).buffer(PluginPartitionAssignment.empty()).build(); + PartitionPlan proposed = PartitionPlan.builder().topic("ranger_audits").version(2).topicPartitionCount(9).putPlugin("hdfs", PluginPartitionAssignment.of(0, 1, 2)).putPlugin("hiveServer2", PluginPartitionAssignment.of(3, 4, 5, 6, 7, 8)).buffer(PluginPartitionAssignment.empty()).build(); + assertDoesNotThrow(() -> PartitionPlanValidator.validateAppendOnly(current, proposed)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java new file mode 100644 index 00000000000..66298e93f3a --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrapTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; +import static org.apache.ranger.audit.server.AuditServerConstants.PROP_SUFFIX_ALLOWED_USERS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ServiceAllowlistBootstrapTest { + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testLoadFromPropertiesParsesServiceRepos() { + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_ozone" + PROP_SUFFIX_ALLOWED_USERS, "om, ozone"); + + Map services = ServiceAllowlistBootstrap.loadAllowlistsFromProperties(props); + + assertEquals(2, services.size()); + assertIterableEquals(List.of("hive"), services.get("dev_hive").getAllowedUsers()); + assertIterableEquals(List.of("om", "ozone"), services.get("dev_ozone").getAllowedUsers()); + } + + @Test + public void testEnrichServicesFromXmlIfMissingMergesInMemoryOnly() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(3) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .build(); + + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + + PartitionPlan enriched = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + + assertEquals(3, enriched.getVersion()); + assertNotNull(enriched.getServices().get("dev_hive")); + PartitionPlanHolder.getInstance().install(enriched, 6); + assertIterableEquals(List.of("hive"), PartitionPlanHolder.getInstance().getAllowedUsersForService("dev_hive")); + } + + @Test + public void testMergeSkippedWhenPlanAlreadyHasPartialServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(services) + .build(); + + Properties props = new Properties(); + props.setProperty(PROP_PREFIX_AUDIT_SERVER_SERVICE + "dev_hive" + PROP_SUFFIX_ALLOWED_USERS, "hive"); + + PartitionPlan merged = ServiceAllowlistBootstrap.mergeSiteXmlAllowlistsWhenPlanServicesMissing(plan, props); + + assertEquals(plan, merged); + assertEquals(1, merged.getServices().size()); + assertNotNull(merged.getServices().get("dev_hdfs")); + assertTrue(merged.getServices().get("dev_hive") == null); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java new file mode 100644 index 00000000000..755a2a78355 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolverTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ServiceAllowlistResolverTest { + private static final Map> STATIC = Map.of( + "dev_hive", Set.of("hive"), + "dev_hdfs", Set.of("hdfs", "nn")); + + @AfterEach + public void tearDown() { + PartitionPlanHolder.getInstance().resetForTests(); + } + + @Test + public void testRejectsBlankServiceOrUser() { + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("", "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser(null, "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testUsesStaticXmlWhenDynamicDisabled() { + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hive", false, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hdfs", false, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testUsesRegistryWhenRepoOnboarded() { + installPartialPlan("dev_hdfs", "hdfs"); + + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "hdfs", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "nn", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testFallsBackToStaticXmlWhenRepoNotInRegistry() { + installPartialPlan("dev_hdfs", "hdfs"); + + assertTrue(ServiceAllowlistResolver.isAllowedServiceUser("dev_hive", "hive", true, PartitionPlanHolder.getInstance(), STATIC)); + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_trino", "trino", true, PartitionPlanHolder.getInstance(), STATIC)); + } + + @Test + public void testDeniesWhenUserNotInRegistryAllowlist() { + Map services = Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs")); + PartitionPlan plan = basePlanBuilder().services(services).build(); + PartitionPlanHolder holder = PartitionPlanHolder.getInstance(); + holder.install(plan, 6); + + assertFalse(ServiceAllowlistResolver.isAllowedServiceUser("dev_hdfs", "nn", true, holder, STATIC)); + } + + private static void installPartialPlan(String repo, String user) { + Map services = new LinkedHashMap<>(); + services.put(repo, ServiceAllowlistEntry.ofUsers(user)); + PartitionPlanHolder.getInstance().install(basePlanBuilder().services(services).build(), 6); + } + + private static PartitionPlan.Builder basePlanBuilder() { + return PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java new file mode 100644 index 00000000000..2778bd833e3 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrap; +import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanBootstrapConfig; +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class PartitionPlanJsonTest { + @Test + public void testRoundTripPreservesPlan() { + PartitionPlan plan = PartitionPlanBootstrap.createInitialPlan(PartitionPlanBootstrapConfig.create("ranger_audits", new String[] {"hdfs", "hiveServer2"}, 3, 9)); + PartitionPlan parsed = PartitionPlan.fromJson(plan.toJson()); + + assertEquals(plan.getTopic(), parsed.getTopic()); + assertEquals(plan.getVersion(), parsed.getVersion()); + assertEquals(plan.getTopicPartitionCount(), parsed.getTopicPartitionCount()); + assertIterableEquals(List.of(0, 1, 2), parsed.getPlugins().get("hdfs").getPartitions()); + assertIterableEquals(List.of(6, 7, 8, 9, 10, 11, 12, 13, 14), parsed.getBuffer().getPartitions()); + } + + @Test + public void testRoundTripPreservesServices() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(2) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive"))) + .build(); + PartitionPlan parsed = PartitionPlan.fromJson(plan.toJson()); + + assertIterableEquals(List.of("hive"), parsed.getServices().get("dev_hive").getAllowedUsers()); + } + + @Test + public void testFromJsonRejectsInvalidPlan() { + assertThrows(PartitionPlanException.class, () -> PartitionPlan.fromJson("{\"topic\":\"ranger_audits\",\"version\":1}")); + } +} From 3073cedc7d8b0c474e48291a36537b9000622579 Mon Sep 17 00:00:00 2001 From: ramk Date: Tue, 23 Jun 2026 19:18:51 +0530 Subject: [PATCH 02/10] RANGER-5655: Tidy ingestor site XML and add PR template. Use hdfs-only allowlist for dev_hdfs, remove unused dev_solr allowlist entry, fix buffer partition example math, and add detailed manual test documentation for PR #1032. Co-authored-by: Cursor --- .../conf/ranger-audit-ingestor-site.xml | 11 +- dev-support/RANGER-5655-PR-TEMPLATE.md | 354 ++++++++++++++++++ 2 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 dev-support/RANGER-5655-PR-TEMPLATE.md diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 6adba2ca1e4..ddb91e36635 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -184,7 +184,7 @@ ranger.audit.ingestor.service.dev_hdfs.allowed.users - hdfs,nn + hdfs Allowed users for dev_hdfs (Policy Manager service name; from policy.download.auth.users) @@ -236,13 +236,6 @@ Allowed users for dev_ozone (Ozone Manager; Ranger plugin runs on OM only) - - ranger.audit.ingestor.service.dev_solr.allowed.users - solr - Allowed users for dev_solr (Solr plugin) - - - ranger.audit.ingestor.auth.to.local @@ -395,7 +388,7 @@ Number of buffer partitions reserved for unconfigured plugins. Used ONLY when configured.plugins is set (for plugin-based partitioning). Total = (sum of plugin partitions) + (buffer partitions). - Example: 2 plugins × 3 + 9 buffer = 15 total. + Example: 7 plugins × 3 + 9 buffer = 30 total. Ignored when configured.plugins is empty (hash-based or dynamic buffer-only bootstrap). diff --git a/dev-support/RANGER-5655-PR-TEMPLATE.md b/dev-support/RANGER-5655-PR-TEMPLATE.md new file mode 100644 index 00000000000..b512c44526a --- /dev/null +++ b/dev-support/RANGER-5655-PR-TEMPLATE.md @@ -0,0 +1,354 @@ +# RANGER-5655 — Pull Request Template + +**GitHub PR:** https://github.com/apache/ranger/pull/1032 +**Jira:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) — Improvement · Major · Fix Version 3.0.0 +**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) + +Copy the section between **START / END** markers into the GitHub PR description. + +--- + +## GitHub PR description (copy from here) + + + +**Jira:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) +**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) +**Full template in repo:** `dev-support/RANGER-5655-PR-TEMPLATE.md` + +## What changes were proposed in this pull request? + +Implements [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655): a **dynamic unified ingestor registry** for Ranger audit-ingestor so operators can change **Kafka partition routing** and **per-repo service allowlists** at runtime — without restarting ingestor pods. + +The registry is a versioned JSON document in Kafka topic `ranger_audit_partition_plan` (**1 partition**, compacted). All ingestor replicas converge via `PartitionPlanWatcher`; `AuditPartitioner` routes on the hot path from in-memory state only. + +**Feature flag (default off):** `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=false` + +### Problem + +| Job | Static behavior today | Pain | +|-----|----------------------|------| +| Service allowlist | `ranger.audit.ingestor.service.*.allowed.users` in site XML at startup | Onboard repo / change allowlist → XML edit + restart all ingestor pods | +| Partition routing | `kafka.configured.plugins` + per-plugin overrides at startup | Promote hot plugin or grow partitions → restart; contiguous ranges can reshuffle later plugins | + +### Solution + +| Component | Role | +|-----------|------| +| `ranger_audit_partition_plan` | Compacted Kafka registry (key = audit topic name, value = plan JSON) | +| `PartitionPlanWatcher` + `PartitionPlanUpdateApplier` | Replay plan topic at startup; poll for updates; install in `PartitionPlanHolder` | +| `AuditPartitioner` | Dynamic mode: round-robin for promoted plugins; sticky hash for buffer; `max(cluster, plan)` after scale when metadata lags | +| `PartitionPlanService` + `AuditREST` | `GET` / `PATCH` / promote / scale / onboard with optimistic locking (`expectedVersion`) | +| `ServiceAllowlistResolver` + `AuthToLocalRuleComposer` | Registry-first allowlist; dynamic `auth_to_local` from allowlist union | +| `KafkaAuditTopicPartitionGrower` | Grows `ranger_audits` before plan references new tail partition ids | +| `PartitionPlanBootstrap` | First pod seeds initial plan from XML when registry is empty | + +**Out of scope for this PR (follow-up):** Solr dispatcher Kerberos config, Docker E2E harness scripts, audit-server README docs. + +--- + +## Code changes (52 files) + +### `audit-common` (3 files) + +| File | Change | +|------|--------| +| `AuditServerConstants.java` | `kafka.partition.plan.*` properties; plan topic constants | +| `AuditMessageQueueUtils.java` | `createPartitionPlanTopicIfNotExists()` — 1 partition, `cleanup.policy=compact` | +| `AuditMessageQueueUtilsTest.java` | Admin client config helper test | + +### `audit-ingestor` — new package `...kafka.partition` (28 production Java) + +| Class | Role | +|-------|------| +| `PartitionPlan`, `PluginPartitionAssignment`, `ServiceAllowlistEntry` | Plan JSON model | +| `OnboardService`, `PromotePlugin`, `PluginScale`, `PartitionPlanReplacement` | REST request DTOs | +| `PartitionPlanHolder` | In-memory plan snapshot (produce hot path) | +| `PartitionPlanWatcher`, `PartitionPlanUpdateApplier` | Kafka plan sync | +| `KafkaPartitionPlanRegistry`, `PartitionPlanRegistry*` | Read/write plan to Kafka | +| `PartitionPlanBootstrap`, `PartitionPlanBootstrapConfig` | XML → initial plan (greenfield) | +| `PartitionPlanAllocator`, `PartitionPlanValidator`, `PartitionPlanRequestValidator` | Append-only promote/scale/replace validation | +| `PartitionPlanService` | REST mutations + topic grow + read-back verify | +| `KafkaAuditTopicPartitionGrower` | AdminClient partition grow | +| `ServiceAllowlistBootstrap`, `ServiceAllowlistResolver` | Allowlist bootstrap/merge + registry-first lookup | +| `AuthToLocalRuleCatalog`, `AuthToLocalRuleComposer`, `PrimaryCatalogRule` | Dynamic auth_to_local from allowlists | +| `PartitionPlanKafkaConfig`, `PartitionPlanConstants` | Config resolution | +| `PartitionPlanException`, `PartitionPlanConflictException` | Error types | + +### `audit-ingestor` — wiring (5 files) + +| File | Change | +|------|--------| +| `AuditPartitioner.java` | Dynamic branch: `partitionFromPlan`, dedicated lanes + buffer pool | +| `AuditMessageQueue.java` | Ensure plan topic; start watcher; bootstrap when registry empty | +| `AuditREST.java` | Partition-plan REST + allowlist delegation | +| `AuditServerConfig.java` | Spring bean for `PartitionPlanService` | +| `ranger-audit-ingestor-site.xml` | Commented dynamic plan property block (default off) | + +### Unit tests (15 files) + +| Test class | Covers | +|------------|--------| +| `AuditPartitionerDynamicTest` | Round-robin, buffer hash, post-scale tail id when metadata lags | +| `PartitionPlanBootstrapTest`, `PartitionPlanBootstrapSupportTest` | Greenfield bootstrap, concurrent cold start | +| `PartitionPlanAllocatorTest` | Promote, scale, append-only tail growth | +| `PartitionPlanValidatorTest` | Duplicate ids, illegal reshuffle | +| `PartitionPlanServiceTest`, `PartitionPlanServiceMutationTest` | REST mutations; **409** on stale `expectedVersion` | +| `PartitionPlanHolderTest`, `PartitionPlanUpdateApplierTest` | Holder install; watcher record apply | +| `PartitionPlanKafkaConfigTest`, `PartitionPlanReplacementTest`, `PartitionPlanJsonTest` | Config + JSON serde | +| `ServiceAllowlistBootstrapTest`, `ServiceAllowlistResolverTest` | Allowlist merge + registry/XML fallback | +| `AuthToLocalRuleComposerTest` | auth_to_local composition from allowlists | + +--- + +## REST API + +Base path: `/api/audit` (same auth as other ingestor APIs). + +| Method | Path | Purpose | +|--------|------|---------| +| `GET` | `/partition-plan` | In-memory plan on this pod | +| `PATCH` | `/partition-plan` | Partial plan replace (`expectedVersion` required) | +| `POST` | `/partition-plan/plugins` | Promote plugin from buffer | +| `PATCH` | `/partition-plan/plugins/{pluginId}` | Scale plugin (append tail partitions) | +| `POST` | `/partition-plan/services` | Onboard repo: allowlist + promote in one version | + +| HTTP status | When | +|-------------|------| +| `200` | Success | +| `400` | Invalid plan / validation failure | +| `409` | Version conflict | +| `503` | Dynamic mode disabled or plan not loaded | + +--- + +## Configuration + +| Property | Default | Description | +|----------|---------|-------------| +| `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` | `false` | Master feature flag | +| `ranger.audit.ingestor.kafka.partition.plan.topic` | `ranger_audit_partition_plan` | Compacted registry topic | +| `ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms` | `30000` | Watcher refresh interval | + +Static XML properties remain used for **initial bootstrap** when the registry is empty. + +--- + +## How was this patch tested? + +### Unit tests + +```bash +mvn test -pl audit-server/audit-common,audit-server/audit-ingestor -Drat.skip=true +``` + +| Gate | Result | +|------|--------| +| `audit-ingestor` partition-plan + allowlist tests | **Pass** (15 test classes in this PR) | +| `AuditPartitionerDynamicTest` | **5/5 pass** — round-robin, buffer, post-scale routing | +| `audit-common` `AuditMessageQueueUtilsTest` | **Pass** | + +Focused run: + +```bash +mvn test -pl audit-server/audit-ingestor \ + -Dtest='AuditPartitionerDynamicTest,PartitionPlan*Test,ServiceAllowlist*Test,AuthToLocalRuleComposerTest' \ + -Drat.skip=true +``` + +### Manual testing (local Docker audit lab) + +Manual validation was done on a **local Docker Compose audit stack** — a developer integration environment that mirrors a real Ranger audit deployment in miniature. It is **not** a production installer; it wires together the same components an operator would run in the field so we can exercise REST APIs, Kafka routing, Kerberos, and full audit delivery end to end. + +**Reference:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) acceptance criteria · [Confluence — Dynamic Ingestor Registry Guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) + +#### What the lab contains + +| Component | Role in testing | +|-----------|-----------------| +| **Kerberos (KDC)** | Plugins and ingestor authenticate with real SPNEGO principals (not mock users) | +| **Kafka** | Audit bus topic `ranger_audits` + registry topic `ranger_audit_partition_plan` (1 partition, compacted) | +| **audit-ingestor** | Receives plugin audits on `:7081`; partition-plan REST API; dynamic routing when flag is on | +| **Second ingestor (optional)** | Replica on `:7082` — proves all pods converge on the same plan without restart | +| **Solr + ZooKeeper** | Audit search backend; verify events appear after full pipeline runs | +| **Audit dispatchers** | Consume `ranger_audits` and write to Solr/HDFS — unchanged by this feature | +| **Ranger Admin + Postgres** | Policy store; Admin **Audit → Access** UI to confirm events | +| **Plugin containers** | HDFS (`ranger-hadoop`), Ozone, Hive, HBase, Kafka authorizer, Knox, KMS — each posts audits to ingestor for its repo | + +**Typical lab layout for partition tests:** 13 configured plugins × 3 partitions each + 9 buffer partitions = **48** partitions on a clean `ranger_audits` topic. Plan watcher refresh interval: **30 seconds** (default). + +**How to bring it up** (companion dev branch — scripts not in this 52-file PR): + +```bash +cd dev-support/ranger-docker +export RANGER_DB_TYPE=postgres +./setup-audit-e2e.sh # full stack: Admin, KDC, Kafka, Solr, ingestor, dispatchers, plugins +./scripts/audit/wait-for-audit-health.sh +``` + +--- + +#### Test block A — Partition plan REST and ingestor behavior + +These tests focus on **ingestor + Kafka only**: turning dynamic mode on/off, the plan registry topic, REST mutations, and multi-pod sync. They do **not** require every plugin pipeline to run. + +**Script:** `verify-partition-plan-e2e.sh`, `verify-partition-plan-e2e-all.sh`, `verify-partition-plan-multipod-e2e.sh`, `verify-partition-plan-brownfield-e2e.sh` + +##### A1. Static mode still works (feature off — default) + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Leave `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=false` (default) | Ingestor starts normally | **Pass** | +| 2 | `GET https://…:7081/api/audit/partition-plan` (Kerberos) | **503** — dynamic plan API not enabled | **Pass** | +| 3 | `GET /api/audit/health` | **200** — ingestor healthy | **Pass** | +| 4 | Trigger HDFS audit (`hdfs dfs -ls` as test user) | Event reaches Solr; Admin Access tab shows new row | **Pass** | + +**Proves:** Existing deployments with the flag off behave exactly as before. No plan topic, no watcher, no REST side effects. + +##### A2. Greenfield — first enable on empty plan topic + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Reset `ranger_audits` to known partition count (48); delete plan topic if present | Clean slate | **Pass** | +| 2 | Set `dynamic.enabled=true`; restart ingestor | Ingestor healthy; logs show `PartitionPlanWatcher` started | **Pass** | +| 3 | `kafka-topics --describe ranger_audit_partition_plan` | Topic exists; **PartitionCount: 1**; `cleanup.policy=compact` | **Pass** | +| 4 | `GET /partition-plan` | JSON **version 1**; 13 plugins listed; `topicPartitionCount` = **48** | **Pass** | +| 5 | Compare plan count vs `kafka-topics --describe ranger_audits` | Numbers **match** | **Pass** | +| 6 | Ingestor startup log | `Mode: dynamic (PartitionPlanHolder)` with plugin ranges and buffer pool | **Pass** | + +**Proves:** On a new cluster, the **first ingestor pod** auto-creates version 1 of the plan from existing XML (`kafka.configured.plugins`, buffer size, per-plugin overrides) — no manual REST call required. + +##### A3. Change routing live via REST (no ingestor restart) + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | `POST /partition-plan/plugins` — promote buffer plugin `storm`, `partitionCount: 2`, correct `expectedVersion` | **200**; version increments; `storm` appears under `plugins` | **Pass** | +| 2 | Same promote with stale `expectedVersion: 1` | **409 Conflict** — forces admin to refresh plan | **Pass** | +| 3 | `POST /plugins` for `hdfs` (already has dedicated partitions) | **400 Bad Request** | **Pass** | +| 4 | `PATCH /partition-plan/plugins/storm` — `additionalPartitions: 1` | **200**; tail partition appended; `ranger_audits` grown via AdminClient if needed | **Pass** | +| 5 | `GET /partition-plan` again **without restart** | Same version and layout as last mutation | **Pass** | + +**Proves:** Operators can promote and scale plugins at runtime. Append-only — existing plugin partition lists are never reshuffled. + +##### A4. Two ingestor pods stay in sync + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Start second ingestor replica on **:7082** (separate hostname + Kerberos identity, same Kafka) | Replica healthy; watcher ready | **Pass** | +| 2 | `GET /partition-plan` on **:7081** and **:7082** | **Same version** at start | **Pass** | +| 3 | Promote a buffer plugin on **primary (:7081) only** | Primary returns new version | **Pass** | +| 4 | Wait **≤ 35 s** (one watcher cycle); `GET /partition-plan` on **:7082** | Replica shows **same new version** — no REST call on replica | **Pass** | + +**Proves:** In a multi-pod cluster behind a load balancer, a routing change propagates to all ingestors automatically within one refresh interval. + +##### A5. Brownfield cutover — pre-load plan before enabling dynamic + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Capture plan JSON while dynamic is briefly on | Valid layout saved | **Pass** | +| 2 | Disable dynamic; delete plan topic | Static mode healthy | **Pass** | +| 3 | Operator pre-seeds plan to Kafka with `updatedBy=brownfield-e2e-seed`, `version: 1` | Message on plan topic | **Pass** | +| 4 | Re-enable dynamic; restart ingestor | Plan shows **brownfield-e2e-seed** — **not** overwritten by XML bootstrap | **Pass** | +| 5 | Rollback: `dynamic.enabled=false` | `GET /partition-plan` → **503**; health **200** | **Pass** | + +**Proves:** Production cutover path works — operator writes the plan first, then enables the feature. + +##### A6. Kafka down at startup (dynamic on) + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Stop Kafka broker; restart ingestor with `dynamic.enabled=true` | Health **not** OK; logs show Kafka / plan watcher failure | **Pass** | +| 2 | Start Kafka; disable dynamic; restart ingestor | Health **200** in static mode | **Pass** | + +**Proves:** Dynamic mode fails visibly when Kafka is unavailable — no silent fallback to a broken state. + +**Block A summary:** **31 automated checks** in the partition-plan script suite — **all passed** (see companion branch validation report). + +--- + +#### Test block B — Full plugin audit pipelines + dynamic routing + +These tests prove the **whole audit path** still works after dynamic partition changes: real plugin operation → Kerberos POST to ingestor → correct Kafka partition → dispatcher → Solr → Admin UI. + +**Scripts:** `verify-hdfs-dynamic-partition-e2e.sh`, `verify-dynamic-partition-plugin-e2e.sh`, `verify-dynamic-auth-to-local-e2e.sh`, `verify-audit-e2e-full.sh` + +##### B1. HDFS — onboard, route, scale, re-route + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Enable dynamic mode (greenfield buffer layout) | Watcher ready | **Pass** | +| 2 | `POST /partition-plan/services` — repo `dev_hdfs`, plugin `hdfs`, allowlist `hdfs`, dedicated partitions | **200**; plan version bumps | **Pass** | +| 3 | Plugin principal POST `/api/audit/access` (SPNEGO) | **200**; `authenticatedUser=hdfs` | **Pass** | +| 4 | Inspect Kafka record for that event | Partition number ∈ `hdfs` assignment list in plan (not buffer) | **Pass** | +| 5 | `PATCH /partition-plan/plugins/hdfs` — add 2 tail partitions | **200**; `ranger_audits` grown; plan lists new ids | **Pass** | +| 6 | Repeat POST `/access` + Kafka inspect | Still lands on an `hdfs` partition from updated plan | **Pass** (routing); Step 5 marker lookup had **one harness flake** on post-scale Kafka poll | +| 7 | Optional: `hdfs dfs -ls /user/` smoke | Full 6-hop: plugin → ingestor → Kafka → dispatcher → Solr → Admin Access | **Pass** when HDFS dispatcher healthy | + +**Proves:** Real HDFS audits follow the live partition plan before and after scale — the core value of dynamic allocation. + +##### B2. Allowlist + auth_to_local (who may POST audits) + +| Step | Action | Expected | Result | +|------|--------|----------|--------| +| 1 | Plugin calls POST `/access` with Kerberos principal (e.g. `hdfs/ranger-hadoop@REALM`) | Ingestor maps via `auth_to_local` → short name `hdfs` → **200** if in `services[dev_hdfs].allowedUsers` | **Pass** | +| 2 | Remove user from allowlist via `PATCH /partition-plan` (services delta only) | Same principal → **403** | **Pass** | +| 3 | Re-add allowlist | **200** again | **Pass** | +| 4 | HDFS principal posting to wrong repo (e.g. `dev_kms`) | **403** cross-repo denial | **Pass** | + +**Proves:** The unified registry `services` map controls authorization without XML restart; `auth_to_local` rules stay aligned with allowlists in dynamic mode. + +##### B3. Other plugins (Ozone, Hive, HBase, Kafka authorizer) + +Same pattern as HDFS: enable dynamic → `POST /partition-plan/services` for repo + plugin + allowlist → POST `/access` with plugin keytab → verify Kafka partition ∈ plan → optional full pipeline to Solr. + +| Plugin | Repo | Principals tested | Pipeline to Solr | +|--------|------|-------------------|------------------| +| Ozone | `dev_ozone` | `om`, `ozone` | **Pass** | +| Hive | `dev_hive` | `hive` | **Pass** | +| HBase | `dev_hbase` | region/master principals | **Pass** | +| Kafka authorizer | `dev_kafka` | kafka plugin principal | **Pass** | + +**Proves:** Dynamic partition plan does not break audit delivery for multiple plugin types. + +--- + +#### What was not manually tested + +| Topic | Why | +|-------|-----| +| Two pods cold-starting **simultaneously** on empty plan topic | Choreographed parallel rollout; covered in unit test `PartitionPlanBootstrapSupportTest` | +| Dedicated **403** for non-admin on partition-plan REST | Deferred — admin allow-list phase | +| Shrinking `ranger_audits` partition count | Not supported by Kafka | +| Production multi-broker failure injection | Single-broker Docker lab only | + +**Note:** Docker E2E scripts live in the full development branch (`dev-support/ranger-docker/scripts/audit/`), not in this 52-file PR. Reproduce using the commands above and the [Confluence guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor). + +### Not in this PR + +| Item | Notes | +|------|-------| +| Solr `useTicketCache=false` (RANGER-5654) | Separate PR | +| Docker E2E script bundle | Follow-up PR | +| audit-server README docs | Follow-up PR | +| Dedicated admin allow-list for partition-plan REST | Deferred | + +--- + +## Backward compatibility + +- Feature flag **off** by default — no new Kafka topic, no watcher, no REST side effects. +- Static `AuditPartitioner` code path unchanged when flag is false. +- Solr/HDFS dispatchers unchanged (consume `ranger_audits` as before). + +--- + +## Checklist + +- [x] [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) linked +- [x] [Confluence guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) referenced +- [x] Feature flag defaults to `false` +- [x] Unit tests pass for `audit-common` + `audit-ingestor` +- [x] Manual Docker audit lab scenarios exercised (partition REST + full plugin pipelines — see template) +- [ ] Reviewer: confirm static-mode regression on your cluster + + From 96b89ee3c10fcacaaa01647c392409ee459a4179 Mon Sep 17 00:00:00 2001 From: ramk Date: Tue, 23 Jun 2026 19:21:10 +0530 Subject: [PATCH 03/10] RANGER-5655: Restore dev_solr allowlist and drop PR template file. Keep dev_solr service allowlist property (remove only the stray blank line the feature commit added). Retain hdfs-only dev_hdfs allowlist and buffer partition example fix. Remove dev-support/RANGER-5655-PR-TEMPLATE.md. Co-authored-by: Cursor --- .../conf/ranger-audit-ingestor-site.xml | 6 + dev-support/RANGER-5655-PR-TEMPLATE.md | 354 ------------------ 2 files changed, 6 insertions(+), 354 deletions(-) delete mode 100644 dev-support/RANGER-5655-PR-TEMPLATE.md diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index ddb91e36635..96ebc7e6aad 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -236,6 +236,12 @@ Allowed users for dev_ozone (Ozone Manager; Ranger plugin runs on OM only) + + ranger.audit.ingestor.service.dev_solr.allowed.users + solr + Allowed users for dev_solr (Solr plugin) + + ranger.audit.ingestor.auth.to.local diff --git a/dev-support/RANGER-5655-PR-TEMPLATE.md b/dev-support/RANGER-5655-PR-TEMPLATE.md deleted file mode 100644 index b512c44526a..00000000000 --- a/dev-support/RANGER-5655-PR-TEMPLATE.md +++ /dev/null @@ -1,354 +0,0 @@ -# RANGER-5655 — Pull Request Template - -**GitHub PR:** https://github.com/apache/ranger/pull/1032 -**Jira:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) — Improvement · Major · Fix Version 3.0.0 -**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) - -Copy the section between **START / END** markers into the GitHub PR description. - ---- - -## GitHub PR description (copy from here) - - - -**Jira:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) -**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) -**Full template in repo:** `dev-support/RANGER-5655-PR-TEMPLATE.md` - -## What changes were proposed in this pull request? - -Implements [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655): a **dynamic unified ingestor registry** for Ranger audit-ingestor so operators can change **Kafka partition routing** and **per-repo service allowlists** at runtime — without restarting ingestor pods. - -The registry is a versioned JSON document in Kafka topic `ranger_audit_partition_plan` (**1 partition**, compacted). All ingestor replicas converge via `PartitionPlanWatcher`; `AuditPartitioner` routes on the hot path from in-memory state only. - -**Feature flag (default off):** `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=false` - -### Problem - -| Job | Static behavior today | Pain | -|-----|----------------------|------| -| Service allowlist | `ranger.audit.ingestor.service.*.allowed.users` in site XML at startup | Onboard repo / change allowlist → XML edit + restart all ingestor pods | -| Partition routing | `kafka.configured.plugins` + per-plugin overrides at startup | Promote hot plugin or grow partitions → restart; contiguous ranges can reshuffle later plugins | - -### Solution - -| Component | Role | -|-----------|------| -| `ranger_audit_partition_plan` | Compacted Kafka registry (key = audit topic name, value = plan JSON) | -| `PartitionPlanWatcher` + `PartitionPlanUpdateApplier` | Replay plan topic at startup; poll for updates; install in `PartitionPlanHolder` | -| `AuditPartitioner` | Dynamic mode: round-robin for promoted plugins; sticky hash for buffer; `max(cluster, plan)` after scale when metadata lags | -| `PartitionPlanService` + `AuditREST` | `GET` / `PATCH` / promote / scale / onboard with optimistic locking (`expectedVersion`) | -| `ServiceAllowlistResolver` + `AuthToLocalRuleComposer` | Registry-first allowlist; dynamic `auth_to_local` from allowlist union | -| `KafkaAuditTopicPartitionGrower` | Grows `ranger_audits` before plan references new tail partition ids | -| `PartitionPlanBootstrap` | First pod seeds initial plan from XML when registry is empty | - -**Out of scope for this PR (follow-up):** Solr dispatcher Kerberos config, Docker E2E harness scripts, audit-server README docs. - ---- - -## Code changes (52 files) - -### `audit-common` (3 files) - -| File | Change | -|------|--------| -| `AuditServerConstants.java` | `kafka.partition.plan.*` properties; plan topic constants | -| `AuditMessageQueueUtils.java` | `createPartitionPlanTopicIfNotExists()` — 1 partition, `cleanup.policy=compact` | -| `AuditMessageQueueUtilsTest.java` | Admin client config helper test | - -### `audit-ingestor` — new package `...kafka.partition` (28 production Java) - -| Class | Role | -|-------|------| -| `PartitionPlan`, `PluginPartitionAssignment`, `ServiceAllowlistEntry` | Plan JSON model | -| `OnboardService`, `PromotePlugin`, `PluginScale`, `PartitionPlanReplacement` | REST request DTOs | -| `PartitionPlanHolder` | In-memory plan snapshot (produce hot path) | -| `PartitionPlanWatcher`, `PartitionPlanUpdateApplier` | Kafka plan sync | -| `KafkaPartitionPlanRegistry`, `PartitionPlanRegistry*` | Read/write plan to Kafka | -| `PartitionPlanBootstrap`, `PartitionPlanBootstrapConfig` | XML → initial plan (greenfield) | -| `PartitionPlanAllocator`, `PartitionPlanValidator`, `PartitionPlanRequestValidator` | Append-only promote/scale/replace validation | -| `PartitionPlanService` | REST mutations + topic grow + read-back verify | -| `KafkaAuditTopicPartitionGrower` | AdminClient partition grow | -| `ServiceAllowlistBootstrap`, `ServiceAllowlistResolver` | Allowlist bootstrap/merge + registry-first lookup | -| `AuthToLocalRuleCatalog`, `AuthToLocalRuleComposer`, `PrimaryCatalogRule` | Dynamic auth_to_local from allowlists | -| `PartitionPlanKafkaConfig`, `PartitionPlanConstants` | Config resolution | -| `PartitionPlanException`, `PartitionPlanConflictException` | Error types | - -### `audit-ingestor` — wiring (5 files) - -| File | Change | -|------|--------| -| `AuditPartitioner.java` | Dynamic branch: `partitionFromPlan`, dedicated lanes + buffer pool | -| `AuditMessageQueue.java` | Ensure plan topic; start watcher; bootstrap when registry empty | -| `AuditREST.java` | Partition-plan REST + allowlist delegation | -| `AuditServerConfig.java` | Spring bean for `PartitionPlanService` | -| `ranger-audit-ingestor-site.xml` | Commented dynamic plan property block (default off) | - -### Unit tests (15 files) - -| Test class | Covers | -|------------|--------| -| `AuditPartitionerDynamicTest` | Round-robin, buffer hash, post-scale tail id when metadata lags | -| `PartitionPlanBootstrapTest`, `PartitionPlanBootstrapSupportTest` | Greenfield bootstrap, concurrent cold start | -| `PartitionPlanAllocatorTest` | Promote, scale, append-only tail growth | -| `PartitionPlanValidatorTest` | Duplicate ids, illegal reshuffle | -| `PartitionPlanServiceTest`, `PartitionPlanServiceMutationTest` | REST mutations; **409** on stale `expectedVersion` | -| `PartitionPlanHolderTest`, `PartitionPlanUpdateApplierTest` | Holder install; watcher record apply | -| `PartitionPlanKafkaConfigTest`, `PartitionPlanReplacementTest`, `PartitionPlanJsonTest` | Config + JSON serde | -| `ServiceAllowlistBootstrapTest`, `ServiceAllowlistResolverTest` | Allowlist merge + registry/XML fallback | -| `AuthToLocalRuleComposerTest` | auth_to_local composition from allowlists | - ---- - -## REST API - -Base path: `/api/audit` (same auth as other ingestor APIs). - -| Method | Path | Purpose | -|--------|------|---------| -| `GET` | `/partition-plan` | In-memory plan on this pod | -| `PATCH` | `/partition-plan` | Partial plan replace (`expectedVersion` required) | -| `POST` | `/partition-plan/plugins` | Promote plugin from buffer | -| `PATCH` | `/partition-plan/plugins/{pluginId}` | Scale plugin (append tail partitions) | -| `POST` | `/partition-plan/services` | Onboard repo: allowlist + promote in one version | - -| HTTP status | When | -|-------------|------| -| `200` | Success | -| `400` | Invalid plan / validation failure | -| `409` | Version conflict | -| `503` | Dynamic mode disabled or plan not loaded | - ---- - -## Configuration - -| Property | Default | Description | -|----------|---------|-------------| -| `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` | `false` | Master feature flag | -| `ranger.audit.ingestor.kafka.partition.plan.topic` | `ranger_audit_partition_plan` | Compacted registry topic | -| `ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms` | `30000` | Watcher refresh interval | - -Static XML properties remain used for **initial bootstrap** when the registry is empty. - ---- - -## How was this patch tested? - -### Unit tests - -```bash -mvn test -pl audit-server/audit-common,audit-server/audit-ingestor -Drat.skip=true -``` - -| Gate | Result | -|------|--------| -| `audit-ingestor` partition-plan + allowlist tests | **Pass** (15 test classes in this PR) | -| `AuditPartitionerDynamicTest` | **5/5 pass** — round-robin, buffer, post-scale routing | -| `audit-common` `AuditMessageQueueUtilsTest` | **Pass** | - -Focused run: - -```bash -mvn test -pl audit-server/audit-ingestor \ - -Dtest='AuditPartitionerDynamicTest,PartitionPlan*Test,ServiceAllowlist*Test,AuthToLocalRuleComposerTest' \ - -Drat.skip=true -``` - -### Manual testing (local Docker audit lab) - -Manual validation was done on a **local Docker Compose audit stack** — a developer integration environment that mirrors a real Ranger audit deployment in miniature. It is **not** a production installer; it wires together the same components an operator would run in the field so we can exercise REST APIs, Kafka routing, Kerberos, and full audit delivery end to end. - -**Reference:** [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) acceptance criteria · [Confluence — Dynamic Ingestor Registry Guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) - -#### What the lab contains - -| Component | Role in testing | -|-----------|-----------------| -| **Kerberos (KDC)** | Plugins and ingestor authenticate with real SPNEGO principals (not mock users) | -| **Kafka** | Audit bus topic `ranger_audits` + registry topic `ranger_audit_partition_plan` (1 partition, compacted) | -| **audit-ingestor** | Receives plugin audits on `:7081`; partition-plan REST API; dynamic routing when flag is on | -| **Second ingestor (optional)** | Replica on `:7082` — proves all pods converge on the same plan without restart | -| **Solr + ZooKeeper** | Audit search backend; verify events appear after full pipeline runs | -| **Audit dispatchers** | Consume `ranger_audits` and write to Solr/HDFS — unchanged by this feature | -| **Ranger Admin + Postgres** | Policy store; Admin **Audit → Access** UI to confirm events | -| **Plugin containers** | HDFS (`ranger-hadoop`), Ozone, Hive, HBase, Kafka authorizer, Knox, KMS — each posts audits to ingestor for its repo | - -**Typical lab layout for partition tests:** 13 configured plugins × 3 partitions each + 9 buffer partitions = **48** partitions on a clean `ranger_audits` topic. Plan watcher refresh interval: **30 seconds** (default). - -**How to bring it up** (companion dev branch — scripts not in this 52-file PR): - -```bash -cd dev-support/ranger-docker -export RANGER_DB_TYPE=postgres -./setup-audit-e2e.sh # full stack: Admin, KDC, Kafka, Solr, ingestor, dispatchers, plugins -./scripts/audit/wait-for-audit-health.sh -``` - ---- - -#### Test block A — Partition plan REST and ingestor behavior - -These tests focus on **ingestor + Kafka only**: turning dynamic mode on/off, the plan registry topic, REST mutations, and multi-pod sync. They do **not** require every plugin pipeline to run. - -**Script:** `verify-partition-plan-e2e.sh`, `verify-partition-plan-e2e-all.sh`, `verify-partition-plan-multipod-e2e.sh`, `verify-partition-plan-brownfield-e2e.sh` - -##### A1. Static mode still works (feature off — default) - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Leave `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=false` (default) | Ingestor starts normally | **Pass** | -| 2 | `GET https://…:7081/api/audit/partition-plan` (Kerberos) | **503** — dynamic plan API not enabled | **Pass** | -| 3 | `GET /api/audit/health` | **200** — ingestor healthy | **Pass** | -| 4 | Trigger HDFS audit (`hdfs dfs -ls` as test user) | Event reaches Solr; Admin Access tab shows new row | **Pass** | - -**Proves:** Existing deployments with the flag off behave exactly as before. No plan topic, no watcher, no REST side effects. - -##### A2. Greenfield — first enable on empty plan topic - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Reset `ranger_audits` to known partition count (48); delete plan topic if present | Clean slate | **Pass** | -| 2 | Set `dynamic.enabled=true`; restart ingestor | Ingestor healthy; logs show `PartitionPlanWatcher` started | **Pass** | -| 3 | `kafka-topics --describe ranger_audit_partition_plan` | Topic exists; **PartitionCount: 1**; `cleanup.policy=compact` | **Pass** | -| 4 | `GET /partition-plan` | JSON **version 1**; 13 plugins listed; `topicPartitionCount` = **48** | **Pass** | -| 5 | Compare plan count vs `kafka-topics --describe ranger_audits` | Numbers **match** | **Pass** | -| 6 | Ingestor startup log | `Mode: dynamic (PartitionPlanHolder)` with plugin ranges and buffer pool | **Pass** | - -**Proves:** On a new cluster, the **first ingestor pod** auto-creates version 1 of the plan from existing XML (`kafka.configured.plugins`, buffer size, per-plugin overrides) — no manual REST call required. - -##### A3. Change routing live via REST (no ingestor restart) - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | `POST /partition-plan/plugins` — promote buffer plugin `storm`, `partitionCount: 2`, correct `expectedVersion` | **200**; version increments; `storm` appears under `plugins` | **Pass** | -| 2 | Same promote with stale `expectedVersion: 1` | **409 Conflict** — forces admin to refresh plan | **Pass** | -| 3 | `POST /plugins` for `hdfs` (already has dedicated partitions) | **400 Bad Request** | **Pass** | -| 4 | `PATCH /partition-plan/plugins/storm` — `additionalPartitions: 1` | **200**; tail partition appended; `ranger_audits` grown via AdminClient if needed | **Pass** | -| 5 | `GET /partition-plan` again **without restart** | Same version and layout as last mutation | **Pass** | - -**Proves:** Operators can promote and scale plugins at runtime. Append-only — existing plugin partition lists are never reshuffled. - -##### A4. Two ingestor pods stay in sync - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Start second ingestor replica on **:7082** (separate hostname + Kerberos identity, same Kafka) | Replica healthy; watcher ready | **Pass** | -| 2 | `GET /partition-plan` on **:7081** and **:7082** | **Same version** at start | **Pass** | -| 3 | Promote a buffer plugin on **primary (:7081) only** | Primary returns new version | **Pass** | -| 4 | Wait **≤ 35 s** (one watcher cycle); `GET /partition-plan` on **:7082** | Replica shows **same new version** — no REST call on replica | **Pass** | - -**Proves:** In a multi-pod cluster behind a load balancer, a routing change propagates to all ingestors automatically within one refresh interval. - -##### A5. Brownfield cutover — pre-load plan before enabling dynamic - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Capture plan JSON while dynamic is briefly on | Valid layout saved | **Pass** | -| 2 | Disable dynamic; delete plan topic | Static mode healthy | **Pass** | -| 3 | Operator pre-seeds plan to Kafka with `updatedBy=brownfield-e2e-seed`, `version: 1` | Message on plan topic | **Pass** | -| 4 | Re-enable dynamic; restart ingestor | Plan shows **brownfield-e2e-seed** — **not** overwritten by XML bootstrap | **Pass** | -| 5 | Rollback: `dynamic.enabled=false` | `GET /partition-plan` → **503**; health **200** | **Pass** | - -**Proves:** Production cutover path works — operator writes the plan first, then enables the feature. - -##### A6. Kafka down at startup (dynamic on) - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Stop Kafka broker; restart ingestor with `dynamic.enabled=true` | Health **not** OK; logs show Kafka / plan watcher failure | **Pass** | -| 2 | Start Kafka; disable dynamic; restart ingestor | Health **200** in static mode | **Pass** | - -**Proves:** Dynamic mode fails visibly when Kafka is unavailable — no silent fallback to a broken state. - -**Block A summary:** **31 automated checks** in the partition-plan script suite — **all passed** (see companion branch validation report). - ---- - -#### Test block B — Full plugin audit pipelines + dynamic routing - -These tests prove the **whole audit path** still works after dynamic partition changes: real plugin operation → Kerberos POST to ingestor → correct Kafka partition → dispatcher → Solr → Admin UI. - -**Scripts:** `verify-hdfs-dynamic-partition-e2e.sh`, `verify-dynamic-partition-plugin-e2e.sh`, `verify-dynamic-auth-to-local-e2e.sh`, `verify-audit-e2e-full.sh` - -##### B1. HDFS — onboard, route, scale, re-route - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Enable dynamic mode (greenfield buffer layout) | Watcher ready | **Pass** | -| 2 | `POST /partition-plan/services` — repo `dev_hdfs`, plugin `hdfs`, allowlist `hdfs`, dedicated partitions | **200**; plan version bumps | **Pass** | -| 3 | Plugin principal POST `/api/audit/access` (SPNEGO) | **200**; `authenticatedUser=hdfs` | **Pass** | -| 4 | Inspect Kafka record for that event | Partition number ∈ `hdfs` assignment list in plan (not buffer) | **Pass** | -| 5 | `PATCH /partition-plan/plugins/hdfs` — add 2 tail partitions | **200**; `ranger_audits` grown; plan lists new ids | **Pass** | -| 6 | Repeat POST `/access` + Kafka inspect | Still lands on an `hdfs` partition from updated plan | **Pass** (routing); Step 5 marker lookup had **one harness flake** on post-scale Kafka poll | -| 7 | Optional: `hdfs dfs -ls /user/` smoke | Full 6-hop: plugin → ingestor → Kafka → dispatcher → Solr → Admin Access | **Pass** when HDFS dispatcher healthy | - -**Proves:** Real HDFS audits follow the live partition plan before and after scale — the core value of dynamic allocation. - -##### B2. Allowlist + auth_to_local (who may POST audits) - -| Step | Action | Expected | Result | -|------|--------|----------|--------| -| 1 | Plugin calls POST `/access` with Kerberos principal (e.g. `hdfs/ranger-hadoop@REALM`) | Ingestor maps via `auth_to_local` → short name `hdfs` → **200** if in `services[dev_hdfs].allowedUsers` | **Pass** | -| 2 | Remove user from allowlist via `PATCH /partition-plan` (services delta only) | Same principal → **403** | **Pass** | -| 3 | Re-add allowlist | **200** again | **Pass** | -| 4 | HDFS principal posting to wrong repo (e.g. `dev_kms`) | **403** cross-repo denial | **Pass** | - -**Proves:** The unified registry `services` map controls authorization without XML restart; `auth_to_local` rules stay aligned with allowlists in dynamic mode. - -##### B3. Other plugins (Ozone, Hive, HBase, Kafka authorizer) - -Same pattern as HDFS: enable dynamic → `POST /partition-plan/services` for repo + plugin + allowlist → POST `/access` with plugin keytab → verify Kafka partition ∈ plan → optional full pipeline to Solr. - -| Plugin | Repo | Principals tested | Pipeline to Solr | -|--------|------|-------------------|------------------| -| Ozone | `dev_ozone` | `om`, `ozone` | **Pass** | -| Hive | `dev_hive` | `hive` | **Pass** | -| HBase | `dev_hbase` | region/master principals | **Pass** | -| Kafka authorizer | `dev_kafka` | kafka plugin principal | **Pass** | - -**Proves:** Dynamic partition plan does not break audit delivery for multiple plugin types. - ---- - -#### What was not manually tested - -| Topic | Why | -|-------|-----| -| Two pods cold-starting **simultaneously** on empty plan topic | Choreographed parallel rollout; covered in unit test `PartitionPlanBootstrapSupportTest` | -| Dedicated **403** for non-admin on partition-plan REST | Deferred — admin allow-list phase | -| Shrinking `ranger_audits` partition count | Not supported by Kafka | -| Production multi-broker failure injection | Single-broker Docker lab only | - -**Note:** Docker E2E scripts live in the full development branch (`dev-support/ranger-docker/scripts/audit/`), not in this 52-file PR. Reproduce using the commands above and the [Confluence guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor). - -### Not in this PR - -| Item | Notes | -|------|-------| -| Solr `useTicketCache=false` (RANGER-5654) | Separate PR | -| Docker E2E script bundle | Follow-up PR | -| audit-server README docs | Follow-up PR | -| Dedicated admin allow-list for partition-plan REST | Deferred | - ---- - -## Backward compatibility - -- Feature flag **off** by default — no new Kafka topic, no watcher, no REST side effects. -- Static `AuditPartitioner` code path unchanged when flag is false. -- Solr/HDFS dispatchers unchanged (consume `ranger_audits` as before). - ---- - -## Checklist - -- [x] [RANGER-5655](https://issues.apache.org/jira/browse/RANGER-5655) linked -- [x] [Confluence guide](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813/Dynamic+Ingestor+Registry+Guide+Ranger+Audit+Ingestor) referenced -- [x] Feature flag defaults to `false` -- [x] Unit tests pass for `audit-common` + `audit-ingestor` -- [x] Manual Docker audit lab scenarios exercised (partition REST + full plugin pipelines — see template) -- [ ] Reviewer: confirm static-mode regression on your cluster - - From fc7ad524b89821dda809ad01d92a30741e18f098 Mon Sep 17 00:00:00 2001 From: ramk Date: Tue, 23 Jun 2026 20:43:22 +0530 Subject: [PATCH 04/10] RANGER-5655: Fix audit-ingestor checkstyle violations for CI. Correct import order, remove unused import, use static requireNonNull, drop duplicate test import, and align PartitionPlan imports with checkstyle rules reported on PR #1032. Co-authored-by: Cursor --- .../producer/kafka/partition/PartitionPlanAllocator.java | 5 +++-- .../kafka/partition/PartitionPlanRequestValidator.java | 2 +- .../audit/producer/kafka/partition/PartitionPlanService.java | 2 +- .../audit/producer/kafka/partition/model/PartitionPlan.java | 3 +-- .../main/java/org/apache/ranger/audit/rest/AuditREST.java | 1 - .../audit/producer/kafka/AuditPartitionerDynamicTest.java | 1 - .../producer/kafka/partition/PartitionPlanBootstrapTest.java | 2 +- .../kafka/partition/PartitionPlanReplacementTest.java | 2 +- .../kafka/partition/PartitionPlanServiceMutationTest.java | 2 +- 9 files changed, 9 insertions(+), 11 deletions(-) diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java index a9456f19353..5bffcadcdfc 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java @@ -30,7 +30,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; + +import static java.util.Objects.requireNonNull; /** Append-only plan updates: promote unknown plugins and scale hot plugins without reshuffling. */ public class PartitionPlanAllocator { @@ -175,7 +176,7 @@ private static boolean pluginHasPartitionCount(PartitionPlan current, String plu } private static void assertPromoteNotConflicting(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { - PluginPartitionAssignment existing = Objects.requireNonNull(current.getPlugins().get(pluginId)); + PluginPartitionAssignment existing = requireNonNull(current.getPlugins().get(pluginId)); if (existing.getPartitions().size() != partitionCount) { throw new PartitionPlanException("Plugin '" + pluginId + "' already has " + existing.getPartitions().size() + " dedicated partition(s); requested " + partitionCount); } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java index f4901e35006..251c89282b0 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java @@ -24,8 +24,8 @@ import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import java.util.List; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java index 53c31b0f47c..591afe8ed03 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -24,8 +24,8 @@ import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java index 845555bea41..40794e976c7 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlan.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.Serializable; import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanValidator; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; import org.apache.ranger.audit.provider.MiscUtil; @@ -36,7 +35,7 @@ @JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) @JsonInclude(JsonInclude.Include.NON_NULL) -public class PartitionPlan implements Serializable { +public class PartitionPlan implements java.io.Serializable { private final String topic; private final int version; private final int topicPartitionCount; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 3f9b9816982..317be6d17d4 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -27,7 +27,6 @@ import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanHolder; import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanKafkaConfig; import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanService; -import org.apache.ranger.audit.producer.kafka.partition.PartitionPlanUpdateApplier; import org.apache.ranger.audit.producer.kafka.partition.ServiceAllowlistResolver; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java index 7cad59f43a7..a45005c9c4d 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/AuditPartitionerDynamicTest.java @@ -34,7 +34,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java index 84c1a3ceef7..d642bcebb08 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanBootstrapTest.java @@ -22,10 +22,10 @@ import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java index fb294867fe9..d8c94698b87 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java @@ -22,8 +22,8 @@ import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; import org.junit.jupiter.api.Test; diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java index ce2ad8b56e7..b2b70290096 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java @@ -23,8 +23,8 @@ import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; +import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.AfterEach; From 1359c1413a89afd8b6955c00b697b489873f604e Mon Sep 17 00:00:00 2001 From: ramk Date: Tue, 23 Jun 2026 21:20:50 +0530 Subject: [PATCH 05/10] RANGER-5655: Seed default configured.plugins list for static partition layout. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship the standard 14-plugin lab list in ranger-audit-ingestor-site.xml with dynamic partition plan disabled by default; update buffer partition example to 14 × 3 + 9 = 51 total. Co-authored-by: Cursor --- .../src/main/resources/conf/ranger-audit-ingestor-site.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 96ebc7e6aad..7f68a020fe7 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -352,7 +352,7 @@ ranger.audit.ingestor.kafka.configured.plugins - + hdfs,yarn,knox,hive,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi Comma-separated plugin IDs (agentId / Kafka record key) that receive dedicated partition ranges in static mode. Leave empty by default; add only the plugins you deploy. @@ -369,7 +369,7 @@ - Dynamic mode: seeds the initial plan from this list when ranger_audit_partition_plan is empty; runtime changes use partition-plan REST, not XML edits. - Example plugin IDs (pick what you run): hdfs, yarn, knox, hiveServer2, hiveMetastore, kafka, + Example plugin IDs (pick what you run): hdfs, yarn, knox, hive, hiveServer2, hiveMetastore, kafka, hbaseRegional, hbaseMaster, solr, trino, ozone, kudu, nifi. Per-plugin partition count: kafka.topic.partitions.per.configured.plugin (default 3), overridable via kafka.plugin.partition.overrides.{pluginId}. @@ -394,7 +394,7 @@ Number of buffer partitions reserved for unconfigured plugins. Used ONLY when configured.plugins is set (for plugin-based partitioning). Total = (sum of plugin partitions) + (buffer partitions). - Example: 7 plugins × 3 + 9 buffer = 30 total. + Example: 14 plugins × 3 + 9 buffer = 51 total. Ignored when configured.plugins is empty (hash-based or dynamic buffer-only bootstrap). From 23be6707f61062d6e11a3e5ca1f16f88180a0165 Mon Sep 17 00:00:00 2001 From: ramk Date: Wed, 24 Jun 2026 12:02:38 +0530 Subject: [PATCH 06/10] RANGER-5655: Simplify partition-plan REST to onboard/update APIs Consolidate partition-plan mutations into three endpoints: GET plan, POST onboard plugin (mandatory non-empty services map), and PATCH update plugin. Remove PATCH /partition-plan and POST /services. Add validator and E2E coverage for mandatory services on onboard. Co-authored-by: Cursor --- ...README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md | 730 ++++++++ ...ADME-KAFKA-PARTITION-PLAN-REGISTRY-REST.md | 1512 +++++++++++++++++ .../partition/PartitionPlanAllocator.java | 202 ++- .../PartitionPlanRequestValidator.java | 90 +- .../kafka/partition/PartitionPlanService.java | 75 +- .../partition/ServiceAllowlistBootstrap.java | 2 +- .../kafka/partition/model/OnboardPlugin.java | 76 + .../model/ServiceAllowlistEntry.java | 23 +- .../kafka/partition/model/UpdatePlugin.java | 92 + .../apache/ranger/audit/rest/AuditREST.java | 97 +- .../partition/PartitionPlanAllocatorTest.java | 78 +- .../partition/PartitionPlanHolderTest.java | 2 +- .../PartitionPlanReplacementTest.java | 31 +- .../PartitionPlanRequestValidatorTest.java | 126 ++ .../PartitionPlanServiceMutationTest.java | 196 +-- .../audit/dynamic-auth-to-local-e2e-lib.sh | 299 ++++ .../audit/dynamic-partition-plugin-e2e-lib.sh | 346 ++++ .../scripts/audit/partition-plan-e2e-lib.sh | 624 +++++++ .../verify-dynamic-partition-plugin-e2e.sh | 138 ++ .../verify-hdfs-dynamic-partition-e2e.sh | 189 +++ .../audit/verify-partition-plan-e2e.sh | 282 +++ 21 files changed, 4867 insertions(+), 343 deletions(-) create mode 100644 audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md create mode 100644 audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java create mode 100755 dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh create mode 100755 dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh create mode 100755 dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh create mode 100755 dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh create mode 100755 dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh create mode 100755 dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh diff --git a/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md b/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md new file mode 100644 index 00000000000..78e0c08139f --- /dev/null +++ b/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md @@ -0,0 +1,730 @@ + + +# Dynamic ingestor registry for Ranger audit-ingestor — guide for everyone + +This guide explains **why** and **how** Ranger moves from **static** ingestor configuration (XML at startup) to a **dynamic unified registry** at runtime — **without restarting** audit-ingestor. + +It covers both: + +1. **Kafka partition routing** — which plugin uses which `ranger_audits` partitions +2. **Service allowlist** — which principals may `POST /api/audit/access` for each Ranger repo + +Both live in one Kafka document on **`ranger_audit_partition_plan`** (`plugins`, `buffer`, and `services`). One REST API: **`/api/audit/partition-plan`**. + +It is written for operators, architects, and reviewers who need a shared mental model — **without reading the codebase**. + +**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813) (child of [Ranger Engineering](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/759726545/Ranger+Engineering)) + +**Related design docs:** [README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md](README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md) · [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) + +## 1. What problem are we solving? + +Ranger plugins (HDFS, Hive, Trino, Ozone OM, etc.) send audit events to **audit-ingestor**, which writes them to Kafka (`ranger_audits`). + +Ingestor performs **two** configuration jobs: + +| Job | Question | Static today | Dynamic goal | +|-----|----------|--------------|--------------| +| **Service allowlist** | May this Kerberos principal POST audits claiming repo `R`? | XML `service..allowed.users` at startup | `services` map in unified registry | +| **Partition routing** | After accept, which Kafka partition? | XML `configured.plugins` at startup | `plugins` / `buffer` in unified registry | + +**Today (static mode):** + +- Allowlist and partition layout are defined in **XML** at startup. +- Adding a repo or plugin usually means: edit XML → **restart ingestor** on every pod. + +**Goal (dynamic mode):** + +- Change allowlist **and** partition assignments **while ingestor is running**. +- Onboard a new plugin/repo without reshuffling existing partition assignments (append-only). +- Keep all ingestor replicas consistent via one shared Kafka compacted topic. + +--- + +## 2. Core ideas (plain language) + +### Kafka topic partitions + +Think of a Kafka topic as a queue split into numbered lanes: partition `0`, `1`, `2`, … + +- Each audit event is routed to **one** partition based on the plugin id (Kafka record **key**). +- More partitions → more parallel consumption downstream (Solr/HDFS dispatchers). + +Kafka allows **increasing** partition count; it does **not** support shrinking. + +### Plugin id + +The plugin id (also called agent id / app id) identifies the source of the audit — e.g. `hdfs`, `hiveServer2`, `trino`. + +### Partition plan (unified registry document) + +A **partition plan** is a versioned JSON document stored in `ranger_audit_partition_plan`. It answers: + +- For each known plugin: **which partition numbers** may receive its audits? (`plugins` + `buffer`) +- For each Ranger service repo: **which short usernames** may `POST /access`? (`services`) +- What is the current **`topicPartitionCount`** (must match Kafka)? + +Example (simplified): + +```json +{ + "topic": "ranger_audits", + "version": 12, + "topicPartitionCount": 48, + "plugins": { + "hdfs": { "partitions": [0, 1, 2, 3, 4, 5] }, + "hiveServer2": { "partitions": [6, 7, 8, 9, 10, 11] } + }, + "buffer": { "partitions": [12, 13, 14, "... through 47 ..."] }, + "services": { + "dev_hive": { "allowedUsers": ["hive"] }, + "dev_ozone": { "allowedUsers": ["om", "ozone"] }, + "dev_trino": { "allowedUsers": ["trino"] } + } +} +``` + +The `version` field increments on every successful admin change (optimistic locking) — **one version** for routing and allowlist mutations. + +Every ingestor pod uses the **same** document so routing and authorization stay consistent. + +### Service allowlist (`POST /api/audit/access`) + +Plugins authenticate (Kerberos/JWT), then ingestor checks `services[serviceName].allowedUsers` (short names after `auth_to_local`). Failure → **403** before any Kafka produce. This check is **orthogonal** to partition routing but stored in the **same** registry document when dynamic mode is on. + +### Unified registry (source of truth) + +The live plan lives in **durable shared storage** that all ingestor replicas read — a Kafka **compacted** topic (`ranger_audit_partition_plan`). + +- No new database or ZooKeeper dependency. +- Survives pod restarts. +- All replicas see the same latest plan. + +### Append-only growth + +When a plugin needs more capacity: + +1. Increase the audit topic’s partition count (add lanes at the **tail**). +2. Assign **only the new** partition numbers to that plugin. +3. **Do not** move partitions away from other plugins. + +### Buffer partitions + +Plugins not yet in the plan (or newly appearing in the fleet) go to **buffer** partitions until an operator **promotes** them to dedicated partitions. + +--- + +## 3. Today vs proposed (at a glance) + +| | Static (today) | Dynamic (unified registry) | +|---|----------------|---------------------------| +| **Where config lives** | XML on each pod at startup | `ranger_audit_partition_plan` (`plugins` + `services`) | +| **Change allowlist or routing** | Edit XML + restart | `/api/audit/partition-plan` REST; no restart | +| **Add new plugin/repo** | XML + restart | `POST .../plugins` (onboard with mandatory `services`) | +| **Scale hot plugin** | Edit overrides + restart | `PATCH .../plugins/{pluginId}` (append-only tail partitions) | +| **Multi-replica ingestor** | Same XML if synced via ConfigMap | All pods watch same Kafka document | +| **Feature flag** | Default behavior | `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=true` | + +--- + +## 4. Static → dynamic cutover — direct answers + +**Goal:** Turn on dynamic mode so each plugin keeps sending audits to the **same Kafka partitions** it uses today — no surprise rerouting on cutover day. + +### When dynamic mode is off + +| Question | Answer | +|----------|--------| +| Is `ranger_audit_partition_plan` created? | **No** — the plan topic is not created or used. | +| How is routing decided? | From XML at startup (static mode), same as today. | +| Is there a background plan sync? | **No**. | + +### When dynamic mode is on + +| Question | Answer | +|----------|--------| +| Is the plan topic created? | **Yes** — on first startup that needs the registry. | +| Where does every ingestor get the plan? | From `ranger_audit_partition_plan` (compacted topic), kept in memory on each pod. | +| Can XML edits change live routing? | **No** (once a plan message exists in Kafka). Runtime changes go through REST. | + +### How “same partitions per plugin” is achieved on cutover + +Ingestor does **not** read Kafka to discover which plugin owns which partition. Kafka stores audit **records**, not plugin ownership. + +Preservation works because the **first published plan** uses the **same layout rules as static mode**: + +| Piece | Static today | Dynamic (first plan from XML) | +|-------|--------------|-------------------------------| +| Plugin order | `configured.plugins` list | Same list | +| Partitions per plugin | Default + per-plugin overrides | Same | +| Layout | Contiguous ranges (hdfs → 0–2, next → 3–5, … buffer tail) | Same ranges as explicit partition ID lists | +| Within-plugin pick | Round-robin per plugin id | Same round-robin | +| Unknown plugin | Hash into buffer | Same buffer pool | + +**What Kafka is used for at bootstrap:** + +| Step | Reads from | Purpose | +|------|------------|---------| +| Build first plan (empty registry) | **XML only** | Plugin list, overrides, buffer → partition lists | +| Install plan (every pod) | **Kafka AdminClient** on `ranger_audits` | **Total** partition count only — must match `topicPartitionCount` in plan | +| Route each audit | **In-memory plan** | No per-event Kafka read | + +**Brownfield (existing production cluster):** Auto-bootstrap from XML is safe only when **XML, ingestor startup logs, and `kafka-topics --describe ranger_audits` all agree**. If they differ, **pre-load the plan into Kafka before enabling dynamic** (operator publishes JSON to the plan topic while dynamic is still off). + +### Plan already in Kafka vs empty registry + +```mermaid +sequenceDiagram + participant Pod as Ingestor pod + participant Plan as ranger_audit_partition_plan + participant Audits as ranger_audits + participant XML as site.xml + + Pod->>Plan: create plan topic if missing (Race A) + Pod->>Plan: read plan for key ranger_audits + alt plan message exists + Plan-->>Pod: use stored plan — skip XML bootstrap + else registry empty + Pod->>XML: build first plan from XML (plugins + services) + Pod->>Plan: re-read (Race B — peer may have published) + Pod->>Plan: publish first plan if still empty + Pod->>Plan: mandatory read-back + end + Pod->>Audits: describe topic — partition count + Pod->>Pod: validate count matches plan → load into memory +``` + +| Situation | What each pod does | +|-----------|-------------------| +| Plan **message** already in `ranger_audit_partition_plan` | Read and use it; **do not** publish a new plan from XML | +| Plan topic exists but **no message** yet | One pod publishes the first plan; others re-read and adopt the same plan (**Race B**) | +| Several pods create the plan **topic** at once | Idempotent create — **Race A**; “already exists” is success | + +**Bootstrap logic (summary):** read plan → if missing, build from XML → re-read → publish if still missing → mandatory read-back → validate partition count → install in memory. + +After the first plan is stored in Kafka, **Kafka is the source of truth** for routing. + +### Multi-pod race — is it implemented? + +| Race | Scenario | Resolution | +|------|----------|------------| +| **A** | Several pods create `ranger_audit_partition_plan` topic together | Idempotent topic create | +| **B** | Several pods publish the **first** plan when registry is empty | Re-read before and after publish; all pods install the same plan from Kafka | + +**Your rule:** *If the plan topic exists but no plan message → add the plan; otherwise use the plan from the topic* — **Yes, implemented** via the bootstrap flow above. + +--- + +## 5. How dynamic mode works (end-to-end) + +| Plane | Kafka topic | Traffic | Who reads/writes | +|-------|-------------|---------|------------------| +| **Data** | `ranger_audits` | High — every audit event | Plugins → ingestor → dispatchers | +| **Control (unified registry)** | `ranger_audit_partition_plan` (compacted) | Low — rare routing + allowlist changes | `/api/audit/partition-plan` REST + `PartitionPlanWatcher` | + +The registry is **configuration**, not audit data. Ingestor keeps the current document **in memory**; only the watcher and REST handlers touch the plan topic. + +### Architecture (control plane vs data plane) + +```mermaid +flowchart TB + subgraph plugins [Ranger plugins] + HS2[HiveServer2] + OM[Ozone OM] + end + + subgraph ops [Ops or automation] + Admin[Admin REST client] + end + + subgraph ingestor [Each audit-ingestor pod] + Access["POST /api/audit/access"] + REST["GET/PATCH/POST /partition-plan"] + Svc[PartitionPlanService] + Watcher[PartitionPlanWatcher] + Mem[(In-memory PartitionPlan
plugins + services)] + Part[Partition router] + Queue[Audit queue] + + REST --> Svc + Svc --> PlanTopic + Watcher -->|poll / consume| PlanTopic + Watcher -->|atomic swap| Mem + HS2 -->|serviceName=dev_hive| Access + OM -->|serviceName=dev_ozone| Access + Access -->|isAllowedServiceUser from services| Mem + Access -->|on 200/202| Queue + Mem --> Part + Queue --> Part + Part -->|key=plugin id| AuditTopic + end + + subgraph kafka [Kafka] + PlanTopic[(ranger_audit_partition_plan
plugins + services)] + AuditTopic[(ranger_audits)] + end + + subgraph consumers [Downstream — unchanged] + Solr[Solr dispatcher] + HDFSdisp[HDFS dispatcher] + end + + Admin -->|via load balancer| REST + Svc -->|createPartitions if needed| AuditTopic + AuditTopic --> Solr + AuditTopic --> HDFSdisp +``` + +### First startup — seeding the plan (once per cluster) + +When dynamic mode starts and the plan registry is **empty**: + +1. Ingestor enables dynamic mode. +2. Watcher finds no document in `ranger_audit_partition_plan`. +3. Ingestor reads XML (`configured.plugins` if set, else buffer-only bootstrap from `kafka.topic.partitions`; overrides, buffer, and `service.*.allowed.users`). +4. Ingestor builds and publishes the **first document** (`plugins` + `services`) to the compacted topic. +5. Ingestor loads that document into memory and begins enforcing allowlist + routing. + +**After that:** additional pods and restarts **read Kafka only** — they do not re-build from XML. + +### Every audit — the hot path + +The plan topic is **not** read on this path. + +```text +Plugin POST /api/audit/access?serviceName=dev_hive&appId=hiveServer2 + │ + ├─ 401 Authentication failed + ├─ 403 services[dev_hive] does not include short username → STOP + └─ 200/202 Allowlist passed → partition router → ranger_audits +``` + +1. Plugin POSTs audit to ingestor (authenticate). +2. `isAllowedServiceUser` reads **in-memory** `services` map (or static XML when dynamic off). +3. Partition router reads **in-memory** `plugins` / `buffer`. +4. Known plugin → round-robin within its partition list; unknown → buffer. +5. Record written to `ranger_audits`. + +### Changing the plan — admin or automation + +**On the pod that receives the REST call:** + +1. Read current plan from Kafka. +2. Validate append-only rules. +3. Grow `ranger_audits` tail if needed (Kafka AdminClient). +4. Write new plan **version** to compacted topic. +5. Return **200 OK** or **409 Conflict** (stale `expectedVersion`). + +**On every ingestor pod (~30s sync interval):** + +1. Background sync picks up new plan version. +2. Validates against live `ranger_audits` partition count. +3. Swaps plan in memory — **no restart**. + +```mermaid +sequenceDiagram + participant Admin as Admin / automation + participant REST as Ingestor REST (one pod) + participant Plan as ranger_audit_partition_plan + participant Audit as ranger_audits + participant W as Background sync (each pod) + participant Mem as In-memory plan + participant Part as Partition router + + Admin->>REST: POST /plugins or PATCH /plugins/{id} + REST->>Plan: read current plan version + REST->>Audit: createPartitions (if needed) + REST->>Plan: write new plan version + REST-->>Admin: 200 OK + + loop Every ingestor pod + W->>Plan: poll latest plan + W->>Mem: atomic swap + Note over Part,Mem: Hot path reads memory only + Part->>Audit: produce audit (key = plugin id) + end +``` + +### Rules to remember + +- **Two topics, two jobs** — plan topic = unified config (`plugins` + `services`); audit topic = data. +- **Memory on the hot path** — no per-audit read of the plan topic. +- **Kafka is the source of truth** after the first document is published. +- **Append-only growth** — new partitions only at the tail of `ranger_audits`. +- **All pods must agree** — every ingestor syncs from the same compacted topic. +- **Allowlist and routing are separate checks** — 403 on `/access` is allowlist; wrong partition is routing. + +--- + +## 6. Admin REST API (control plane) + +When dynamic mode is on, operators change routing **and** allowlists through **`/api/audit/partition-plan`** on **any** pod (usually via load balancer). Mutations are written to `ranger_audit_partition_plan`; every pod picks up changes through `PartitionPlanWatcher` (~30s). + +**Auth:** Kerberos or JWT. When `kafka.partition.plan.allowed.users` is set, only those short names may call partition-plan REST (plugin users must not). Dynamic mode off → all partition-plan calls return **503**. + +### Endpoints (three only) + +| Method | Path | Purpose | +|--------|------|---------| +| `GET` | `/api/audit/partition-plan` | Read current plan (`plugins`, `buffer`, `services`, `version`) | +| `POST` | `/api/audit/partition-plan/plugins` | **Onboard** plugin: dedicated partitions + service allowlists (one version bump) | +| `PATCH` | `/api/audit/partition-plan/plugins/{pluginId}` | **Update** onboarded plugin: scale and/or service allowlist mutations | + +Base URL example: `https://:7081/api/audit/partition-plan` + +All mutations require **`expectedVersion`** from the last `GET`. Stale version → **409 Conflict** + current plan in body. + +**Removed (consolidated above):** `PATCH /api/audit/partition-plan`, `POST /api/audit/partition-plan/services`, separate promote-only / scale-only flows. + +> **Future work (not implemented):** bootstrap **v0** split — seeding `services` from XML separately from plugin partition assignments. Current bootstrap still publishes a single v1 plan from XML. + +### Common operations + +**Read plan** + +```http +GET /api/audit/partition-plan +→ 200 + JSON plan (note the "version" field) +``` + +**Onboard plugin** — dedicated partitions + allowlists in one call (`services` is **required** and must be non-empty): + +```json +POST /api/audit/partition-plan/plugins +{ + "pluginId": "hiveServer2", + "partitionCount": 3, + "expectedVersion": 1, + "services": { + "dev_hive": { "allowedUsers": ["hive"] }, + "dev_hive2": { "allowedUsers": ["hive2"] } + } +} +``` + +Each service entry is stored with `pluginId` for ownership tracking. + +**Update plugin** — scale and/or mutate allowlists scoped to `{pluginId}` (at least one delta required): + +```json +PATCH /api/audit/partition-plan/plugins/hiveServer2 +{ + "expectedVersion": 2, + "addServices": { "dev_hive3": { "allowedUsers": ["hive3"] } }, + "removeServices": ["dev_hive2"], + "additionalPartitions": 2 +} +``` + +| Field | Purpose | +|-------|---------| +| `additionalPartitions` | int ≥ 1 — append tail partition IDs (append-only) | +| `addServices` | map repo → `{ "allowedUsers": [...] }` | +| `updateServices` | map repo → `{ "allowedUsers": [...] }` — replace allowlist for repos owned by `{pluginId}` | +| `removeServices` | list of repo names to remove (scoped to `{pluginId}`) | + +Success → **200** + updated plan JSON (version incremented on change). +Repeating the same onboard or update delta (with matching `expectedVersion`) → **200** + current plan JSON with **no** registry write or version bump. +State conflict (resource exists but request differs, e.g. different partition count) → **400**. +Stale version → **409** + current plan in body. + +### What happens inside one REST call + +```mermaid +flowchart LR + Admin[Admin or script] --> REST[Ingestor REST] + REST --> Read[Read plan from Kafka] + Read --> Check{expectedVersion OK?} + Check -->|No| R409[409 + current plan] + Check -->|Yes| Valid[Validate append-only change] + Valid --> Grow{Need more audit partitions?} + Grow -->|Yes| Topic[Grow ranger_audits tail] + Grow -->|No| Write[Write new plan to Kafka] + Topic --> Write + Write --> OK[200 + new plan] + OK --> Sync[All pods sync within ~30s] +``` + +| Step | What the ingestor does | +|------|------------------------| +| 1 | Authenticate caller; check `kafka.partition.plan.allowed.users` when configured | +| 2 | Read current plan from `ranger_audit_partition_plan` | +| 3 | Reject if `expectedVersion` does not match | +| 4 | Compute new plugin lists (append-only — no reshuffling existing slots) | +| 5 | If new partition IDs are needed → grow `ranger_audits` **first** | +| 6 | Publish new plan version to Kafka | +| 7 | Return updated plan JSON | + +**GET** is cheap (memory). **POST / PATCH** always goes through Kafka so all pods converge on the same plan. + +--- + +## 7. Operator workflow: onboarding a plugin or repo + +**Recommended:** one `POST /api/audit/partition-plan/plugins` with a **non-empty `services` map** — allowlist and dedicated partitions in one plan version. + +### Stage 0 — Create service in Ranger Admin + +1. Create service `dev_trino` in Policy Manager; set `policy.download.auth.users`. +2. Configure plugin audit destination → ingestor URL (`:7081`). + +### Stage 1 — Onboard via unified registry + +```http +POST /api/audit/partition-plan/plugins +``` + +Include `pluginId`, `partitionCount`, `expectedVersion`, and at least one repo in `services` (see [§6](#6-admin-rest-api-control-plane)). + +All ingestors apply within ~30s. **No restart** required. + +### Stage 2 — Verify plugin POST + +Expect **200/202** on `/access`, not **403**. + +### Stage 3 — Scale or change allowlists (optional) + +Call `PATCH /api/audit/partition-plan/plugins/{pluginId}` to add tail partitions, add/update/remove repos, or combine all in one version bump. + +**Do not** edit `ranger-audit-ingestor-site.xml` on one pod for runtime changes. XML is only for **initial bootstrap** when the registry is empty. + +--- + +## 8. Operator workflow: partition-only changes + +Use when an onboarded plugin already has allowlists and you only need routing changes (scale tail partitions). + +### Stage 0 — Plugin appears (unknown) + +- Audits use **buffer** partitions. +- Monitor volume per plugin id. + +### Stage 1 — Onboard plugin (partitions + allowlists) + +- Call `POST /api/audit/partition-plan/plugins` with mandatory `services` (see [§6](#6-admin-rest-api-control-plane)). +- All ingestors apply within ~30s. **No restart** required. + +### Stage 2 — Scale a hot plugin + +- Call `PATCH /api/audit/partition-plan/plugins/{pluginId}` with `additionalPartitions`. +- Dispatchers rebalance automatically when the audit topic grows. + +**Do not** edit `ranger-audit-ingestor-site.xml` on one pod for runtime changes. XML is only for **initial bootstrap** when the plan registry is empty. + +--- + +## 9. Configuration (dynamic mode) + +| Property | Purpose | Example | +|----------|---------|---------| +| `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` | Turn dynamic unified registry on/off | `false` (default) = static XML | +| `ranger.audit.ingestor.kafka.partition.plan.topic` | Compacted registry topic name | `ranger_audit_partition_plan` | +| `ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms` | How often pods reload plan | `30000` | +| `ranger.audit.ingestor.kafka.partition.plan.allowed.users` | Who may call partition-plan REST | `admin,ops` | +| `ranger.audit.ingestor.service..allowed.users` | Static bootstrap per-repo allowlist | `hive`, `om,ozone`, … | +| `ranger.audit.ingestor.auth.to.local` | Principal → short name rules | Same as Hadoop `hadoop.security.auth_to_local` | + +When dynamic is **off**, routing and allowlist are fixed from XML at startup; the plan topic is not used. + +When dynamic is **on** and the registry is **empty**, the first ingestor pod seeds the plan (`plugins` + `services`) from XML. Later pods and restarts read **Kafka only**. + +--- + +## 10. FAQ + +### Basics + +**Why plugin-based partitioning?** +Hot plugins (HDFS, Hive) can get dedicated Kafka lanes so they do not starve others. Unknown plugins share a buffer until you promote them. + +**What is the difference between static and dynamic mode?** +Static: routing is computed once from XML at startup; changes need restart. Dynamic: routing lives in Kafka; admins change it via REST while ingestor keeps running. + +**Why not store the plan in Postgres or ZooKeeper?** +Ingestor already requires Kafka. A compacted plan topic adds no new infrastructure. + +**Why not edit XML on a running pod?** +Each pod has its own copy; edits are not shared, not durable, and are lost on restart. Runtime changes belong in the plan topic via REST. + +**Can I change routing by editing XML while dynamic mode is on?** +No for live routing. Ingestor uses the Kafka plan in memory, not XML edits on disk. Update XML only when preparing static rollback or documenting the intended layout. + +**Can I decrease `ranger_audits` partition count?** +No. Kafka does not support shrinking partitions. You can only add partitions at the tail. + +### Two topics and sync + +**What are the two Kafka topics?** +`ranger_audits` = audit data (high volume). `ranger_audit_partition_plan` = routing config (low volume, compacted). + +**Does every audit POST read the plan topic?** +No. Each audit uses the plan already in **memory** on that pod. Only background sync and REST mutations touch the plan topic. + +**How do all ingestor pods stay in sync?** +Every pod watches the same compacted plan topic (default every 30s) and swaps the new plan into memory. + +**What happens when a pod restarts?** +It reads the latest plan from Kafka (if dynamic is on). The plan survives in Kafka across crashes. + +**Do Solr and HDFS dispatchers need the partition plan?** +No. They consume **all** partitions of `ranger_audits`. Only ingestor uses the plan to **choose** which partition to write to. + +**Will changing the plan break consumers?** +Adding partitions triggers normal consumer rebalance. Existing plugin slots keep the same partition numbers if you follow append-only promote/scale rules. + +### Plan content and routing + +**What is the buffer?** +Partitions reserved for plugins that are not yet promoted (or newly seen plugin ids). Promote moves a plugin from buffer to dedicated slots. + +**What does append-only mean?** +When scaling, only **new** partition numbers at the end of `ranger_audits` are assigned. Existing plugins keep the same partition IDs in the same order. + +**How is a partition chosen inside a plugin’s list?** +Round-robin per plugin id — same behavior as static mode. + +**Where does an unknown plugin send audits?** +To the buffer partition pool (hash-based pick within buffer list in dynamic mode). + +### REST and concurrency + +**Do I need to restart ingestor after promote or scale?** +No. Background sync applies the new plan within about one refresh interval (~30s). + +**What is `expectedVersion`?** +The plan `version` you believe is current when you write. If someone else published first, your version is stale and you get **409**. + +**What should I do on HTTP 409?** +Another writer published a newer plan. Use the plan in the 409 response (or `GET` again), note the new `version`, and retry your change with that `expectedVersion`. + +**Why grow `ranger_audits` before publishing a new plan?** +The plan must not reference partition IDs that do not exist yet. The server grows the audit topic tail first, then writes the plan. + +**Why does promote return 400?** +Common cases: plugin already has dedicated partitions, invalid partition count, or plugin id missing. Scale returns 400 if the plugin is not in the plan yet (promote it first). + +**What do the HTTP status codes mean?** +**503** — dynamic mode is off, or the server could not grow `ranger_audits` (Kafka admin failure). **400** — validation failed (bad shape, append-only violation, `topicPartitionCount` ≠ Kafka). **409** — version conflict; retry with the plan body returned in the response. + +**Do dispatchers need reconfiguration after scale?** +No. Consumer groups rebalance automatically when partition count increases. Tune consumer threads only if you see sustained lag. + +### Cutover and bootstrap + +**When is `ranger_audit_partition_plan` created?** +Only when dynamic mode is enabled. With dynamic off, the topic is not created or used. + +**Does bootstrap read Kafka to learn plugin → partition mapping?** +No. The first plan is built from XML (same layout as static mode). Kafka is used for **total** partition count validation and as the durable registry. + +**Can I publish the plan to Kafka before enabling dynamic mode?** +Yes — recommended for brownfield clusters. Ingestor will read your pre-loaded plan and will not replace it with a fresh XML bootstrap. + +**Greenfield vs brownfield cutover — what is different?** +Greenfield: enable dynamic on an empty registry; first pod seeds from XML (empty `configured.plugins` → buffer-only plan sized by `kafka.topic.partitions`). Brownfield: export static layout, pre-seed the plan topic (or set `configured.plugins` in XML to match production), then enable dynamic and verify every pod shows the same plan. + +**Does the `configured.plugins` XML list still matter in dynamic mode?** +Yes for **bootstrap only** when the registry is empty. After the first plan exists, runtime routing changes go through REST, not by editing that list. + +**How do I verify cutover succeeded?** +`GET /api/audit/partition-plan` on every pod (same `version`); plugin lists match your saved static logs; `topicPartitionCount` matches `kafka-topics --describe ranger_audits`. + +**How do I roll back to static mode?** +Export current plan via `GET`, align XML to that layout if needed, set `dynamic.enabled=false`, rolling restart. Plan topic remains but is ignored. + +**What if multiple pods start together with an empty registry?** +Race A: idempotent plan-topic create. Race B: re-read before/after first publish; all pods install the same plan. See [§4](#4-static--dynamic-cutover--direct-answers). + +### Troubleshooting + +**Ingestor fails startup after enabling dynamic — what now?** +Often Kafka unreachable or cannot create/read the plan topic. Fix Kafka connectivity; keep dynamic off until Kafka is healthy. With Kafka down at startup, ingestor should **not** come up healthy in dynamic mode. + +**Pods show different plan versions — what now?** +Wait one refresh interval after a change. If still mismatched, check plan topic readability and watcher logs on the lagging pod. + +**Does audit recovery / local spool behavior change?** +No. Per-pod spool and retry when Kafka is briefly unavailable works as today. + +**Does this replace producer throughput tuning?** +No. Batch size, linger, and compression are separate settings. + +**Who can call partition-plan REST?** +Ops principals in `kafka.partition.plan.allowed.users` when configured (Kerberos/JWT). Plugin users (`hive`, `om`) send audits via `/access` — they must **not** mutate the registry. + +### Service allowlist (`POST /api/audit/access`) + +**Why do we need an allowlist if Kerberos already authenticates the plugin?** +Authentication proves *who* connected. Authorization proves they may **claim audits for this repo**. Kerberos success alone would let any daemon POST as any `serviceName`. + +**Does dynamic partition plan remove the allowlist check?** +**No.** Partition routing runs **after** `/access` accepts the batch. **403** on `/access` is always a **service allowlist** failure (not routing). + +**What is `serviceName`?** +The Ranger Policy Manager **repo name** (e.g. `dev_hive`), not the service type (`hive`). + +**What goes in `allowedUsers`?** +Short names after `auth_to_local` — same values as `policy.download.auth.users` on the Ranger service (or a subset). + +**Consistency rule (ops discipline):** +`allowedUsers for repo R ⊆ { short names from policy.download.auth.users for R in Ranger Admin }`. Ranger Admin does not auto-sync in Phase 1; ingestor may reject REST writes that violate subset rules (strict mode). + +**Three layers — do not merge:** + +| Layer | Who | Purpose | +|-------|-----|---------| +| **Service allowlist** (plugin POST) | Daemons (`hive`, `om`, …) | May this principal POST audits for repo `R`? | +| **Partition plan** (Kafka routing) | Ingestor internal | Which Kafka partition after accept? | +| **Admin REST APIs** | Ops (`admin`, `ops`, …) | Who may change the unified registry via `/api/audit/partition-plan`? | + +**If you see…** + +| Symptom | Layer | Fix | +|---------|-------|-----| +| **401** on `/access` | Authentication | Plugin / ingestor Kerberos, keytabs, SPNEGO | +| **403** on `/access` | **Service allowlist** | Add repo via `POST .../plugins` (onboard) or `PATCH .../plugins/{pluginId}` (`addServices` / `updateServices`) | +| Audits accepted but wrong Kafka partition | **Partition plan** | `POST .../plugins` (onboard) / `PATCH .../plugins/{pluginId}` (scale) | +| New repo in Admin, audits still **403** | Allowlist not onboarded | `POST .../plugins` or `PATCH .../plugins/{pluginId}` with `addServices` | +| Allowlist OK, audits in buffer partition | Partition plan not onboarded | `POST .../plugins` with `partitionCount` + `services` | + +**Plugin gets 403 but Kerberos works — what now?** +Short name not in `services[repo].allowedUsers`, wrong `serviceName`, or `auth_to_local` mismatch. + +**New repo in Admin UI, still 403?** +Creating the service in Admin does not auto-update ingestor (Phase 1). Call `POST /api/audit/partition-plan/plugins` (new plugin) or `PATCH .../plugins/{pluginId}` with `addServices`. + +**Do I need both allowlist and partition plan for a new plugin?** +Yes for full production onboarding. `POST /api/audit/partition-plan/plugins` requires a non-empty `services` map and assigns dedicated partitions in **one** plan version. + +**Is there a bundled API?** +`POST /api/audit/partition-plan/plugins` registers `services[repo]` allowlists and promotes plugin partitions atomically. + +--- + +## 11. Docs + +| Doc | Purpose | +|-----|---------| +| [README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md](README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md) | Service allowlist design (unified `services` map) | +| [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) | Partition plan architecture | +| [README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md](README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md) | Partition-plan Kafka topic + REST | +| [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) | Partition-plan implementation phases | +| [DESIGN-KAFKA-AUDIT-SERVER.md](DESIGN-KAFKA-AUDIT-SERVER.md) | End-to-end audit pipeline | +| [PR #1017](https://github.com/apache/ranger/pull/1017) (RANGER-5645) | Static Docker allowlist fix | diff --git a/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md b/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md new file mode 100644 index 00000000000..aac724dbfa6 --- /dev/null +++ b/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md @@ -0,0 +1,1512 @@ + + +# Kafka-backed partition plan registry + REST (detailed design) + +> **Consolidated design (start here for review):** [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) — architecture, flows, diagrams, brownfield migration, and Q&A in one document. This README is the deep dive on Kafka registry + REST semantics. + +This document describes the **recommended** approach for **dynamic plugin onboarding** and **dynamic partition scaling** in `audit-ingestor` without Postgres or ZooKeeper. + +**Related docs:** + +- **Consolidated design:** [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) +- Current static partitioning: `README-KAFKA-PLUGIN-PARTITIONING.md` +- Design summary + checklist: `README-KAFKA-PLUGIN-PARTITIONING-DYNAMIC-DESIGN.md` +- **Phased implementation plan:** [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) +- Producer performance gaps: `README-KAFKA-PRODUCER-PERFORMANCE.md` +- Solr/HDFS consumers: `README-KAFKA-DISPATCHERS.md` + +--- + +## Problem we are solving + +Today (`README-KAFKA-PLUGIN-PARTITIONING.md`): + +- Plugin → partition mapping is computed once at ingestor startup from XML. +- Changing `configured.plugins` or overrides requires restart. +- Contiguous-range allocation **reshuffles** later plugins when an early plugin’s count changes. + +Goal: + +- **Onboard** plugins (e.g. trino) and **scale** hot plugins (e.g. hiveServer2) **without restart**. +- **No new infra** (no Postgres/ZooKeeper). +- **All ingestor replicas** use the same routing after a change. +- **Survive pod crash/restart** without losing the plan. + +--- + +## High-level architecture + +### In plain terms + +There are **two separate Kafka topics** and **two different jobs**: + +| Topic | What it stores | Who cares | +|-------|----------------|-----------| +| **`ranger_audit_partition_plan`** | Small JSON: which plugin uses which partition IDs | **Ingestor only** (config) | +| **`ranger_audits`** | Actual audit events | **Ingestor produces**, **Solr/HDFS consume** | + +**Ingestor pods do three things:** + +1. **Serve audits** — plugins POST `/access` → pick partition from **in-memory plan** → write to `ranger_audits` (fast; no Kafka read per audit). +2. **Watch the plan** — background thread reads `ranger_audit_partition_plan` and updates memory when the plan changes. +3. **Admin REST** — ops call `GET /api/audit/partition-plan`, `POST .../plugins` (onboard), or `PATCH .../plugins/{pluginId}` (update). + +Solr and HDFS dispatchers **never read the plan**. They read **all** partitions of `ranger_audits`. + +--- + +### Picture (control plane vs data plane) + +```mermaid +flowchart TB + subgraph ops [Ops / automation] + Admin[Admin REST client] + end + + subgraph ingestor [audit-ingestor pods A, B, C ...] + REST[AuditREST partition-plan API] + Svc[PartitionPlanService] + Watcher[PartitionPlanWatcher background thread] + Mem[(In-memory PartitionPlan)] + Part[AuditPartitioner] + Queue[AuditMessageQueue] + + REST --> Svc + Svc --> PlanTopic + Watcher --> PlanTopic + Watcher --> Mem + Mem --> Part + Plugin[Plugin POST /access] --> Queue --> Part + Part --> AuditTopic + end + + subgraph kafka [Kafka] + PlanTopic[(ranger_audit_partition_plan compacted)] + AuditTopic[(ranger_audits)] + end + + subgraph consumers [Unchanged consumers] + Solr[Solr dispatcher] + HDFS[HDFS dispatcher] + end + + Admin -->|any pod via LB| REST + Svc -->|createPartitions if needed| AuditTopic + AuditTopic --> Solr + AuditTopic --> HDFS +``` + +**Takeaway:** plan topic = **slow, rare config**; audit topic = **high volume data**. Only REST + watcher touch the plan topic. + +--- + +### The three flows (step by step) + +--- + +#### Flow 1 — Admin changes the plan + +**When:** You onboard a plugin (e.g. trino) or give a hot plugin more partitions (e.g. hiveServer2). +**How often:** Rare — ops or automation, not every audit. + +| Step | What happens | +|------|----------------| +| 1 | Admin sends REST to **any** ingestor pod (`GET` to read, `POST .../plugins` to onboard, `PATCH .../plugins/{pluginId}` to update). | +| 2 | That pod reads the current plan from **`ranger_audit_partition_plan`** (e.g. version 4). | +| 3 | If more partitions are needed, pod grows **`ranger_audits`** via Kafka AdminClient. | +| 4 | Pod writes the **new plan** (version 5) to the compacted topic. | +| 5 | Pod returns **200 OK** to admin (or **409** if another pod updated first — retry with new version). | +| 6 | **Every ingestor pod** — background watcher loads version 5 into memory (~30s or on Kafka message). | +| 7 | **Solr / HDFS** — no config change; they rebalance only if `ranger_audits` got more partitions. | + +```mermaid +flowchart TB + Admin[Admin / automation] + + subgraph step1 [Ingestor pod handles REST] + REST[Ingestor REST] + PlanRW[(Plan topic
read + write)] + Grow[(ranger_audits
grow if needed)] + REST --> PlanRW + REST --> Grow + end + + subgraph step2 [All ingestor pods sync] + Watcher[Watcher on each pod] + Mem[Memory updated] + Same[Same routing everywhere] + Watcher --> Mem --> Same + end + + subgraph step3 [Dispatchers unchanged] + SolrHDFS[Solr / HDFS] + Consume[Keep consuming ranger_audits] + SolrHDFS --> Consume + end + + Admin --> REST + PlanRW --> Watcher + Grow -.->|rebalance only if partitions grew| SolrHDFS +``` + +--- + +#### Flow 2 — Plugin sends an audit + +**When:** Every time a Ranger plugin reports an access event. +**How often:** High volume — this is the hot path. + +| Step | What happens | +|------|----------------| +| 1 | Plugin sends **POST `/access`** to ingestor (any pod via load balancer). | +| 2 | Ingestor reads the plan **already in memory** (not from Kafka). | +| 3 | Ingestor finds partitions for this plugin id (`agentId`), e.g. hive → `[4,5,6,7,8,9]`. | +| 4 | Ingestor picks one partition (round-robin within that list). | +| 5 | Ingestor writes the audit event to **`ranger_audits`**. | + +**Important:** This path **never** reads `ranger_audit_partition_plan`. Fast and cheap. + +```mermaid +flowchart LR + Plugin[Plugin] --> Access[POST /access] + Access --> Mem[(Memory
plan)] + Mem --> Pick[Pick partition
for agentId] + Pick --> Audit[(ranger_audits)] + + Watcher[Watcher
Flow 1 or Flow 3] -.->|fills memory| Mem +``` + +--- + +#### Flow 3 — Pod startup (first pod vs others) + +**When:** Ingestor pod starts or restarts. +**Goal:** Every pod ends up with the **same plan in memory**. + +**Prerequisite for XML → Kafka bootstrap:** `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=true`. If `false` or absent, legacy XML-only mode applies and **`ranger_audit_partition_plan` is not populated**. + +| Situation | What the pod does | +|-----------|-------------------| +| **Pod 1** — plan topic is **empty** | Read XML → build plan **v1** → **write v1 to `ranger_audit_partition_plan`** → load into memory. | +| **Pod 2, 3, …** — plan **already in Kafka** | **Read plan from Kafka only** → load into memory. **Do not** use XML for routing. | +| **Any pod restarts** | Same as Pod 2+ — read from Kafka, not XML. | + +```mermaid +flowchart TB + subgraph pod1 [First pod ever — plan topic empty] + direction LR + XML[XML config] --> Kafka1[(ranger_audit_partition_plan v1)] + Kafka1 --> Mem1[Memory] + end + + subgraph podN [Later pods — plan already in Kafka] + direction LR + Kafka2[(ranger_audit_partition_plan)] --> Mem2[Memory] + XML2[XML ignored for routing] + XML2 -.->|not used| Mem2 + end + + subgraph podR [Pod crash or restart] + direction LR + Kafka3[(ranger_audit_partition_plan
plan survives in Kafka)] --> Mem3[Memory] + end +``` + +**Rule of thumb:** After v1 exists in Kafka, **Kafka is the source of truth** — not XML on the pod. + +**Full bootstrap details** (XML properties, example v1 JSON, when bootstrap does *not* run): [Bootstrap and multi-pod startup → First pod: XML populates the plan topic](#first-pod-xml-populates-the-plan-topic). + +--- + +### Who talks to what (quick reference) + +| Actor | Reads plan topic | Writes plan topic | Reads/writes ranger_audits | +|-------|------------------|-------------------|----------------------------| +| Plugin audit POST | No | No | Produce only | +| PartitionPlanWatcher | Yes (background) | No | No | +| REST GET/POST/PATCH | Yes | Yes (POST/PATCH mutations) | AdminClient grow only | +| Solr dispatcher | No | No | Consume all partitions | +| HDFS dispatcher | No | No | Consume all partitions | + +--- + +## Code map: design vs `audit-server` today + +The dynamic partition plan is **design + README only** — the classes below marked **Proposed** do not exist in the repo yet. This section maps the design to **current audit-ingestor code** so you can see what to extend. + +### Module layout + +| Module | Path | Role | +|--------|------|------| +| **audit-ingestor** | `audit-server/audit-ingestor/` | HTTP REST, Kafka producer, recovery | +| **audit-common** | `audit-server/audit-common/` | Shared constants, `AuditMessageQueueUtils` (AdminClient) | +| **audit-dispatcher** | `audit-server/audit-dispatcher/` | Solr/HDFS consumers (unchanged by partition plan) | + +### Design component → Java / config (exists today vs proposed) + +| Design component | Status | Location in repo | +|------------------|--------|------------------| +| **`PartitionPlan`** (JSON model) | **Proposed** | New class, e.g. `audit-ingestor/.../partition/PartitionPlan.java` | +| **`PartitionPlanAllocator`** | **Proposed** | New — append-only tail allocation | +| **`PartitionPlanRegistry`** | **Proposed** | New — Kafka compacted topic read/write | +| **`PartitionPlanService`** | **Proposed** | New — REST handler logic | +| **`PartitionPlanWatcher`** | **Proposed** | New — background thread per pod | +| **`AuditPartitioner`** | **Exists** (static XML) | `audit-ingestor/.../kafka/AuditPartitioner.java` | +| **`AuditREST`** `/access` | **Exists** | `audit-ingestor/.../rest/AuditREST.java` | +| **`AuditREST`** `/partition-plan` | **Proposed** | Same file — new `@GET` / `@PATCH` / `@POST` methods | +| **`AuditMessageQueue`** | **Exists** | `audit-ingestor/.../kafka/AuditMessageQueue.java` | +| **`AuditProducer`** | **Exists** | `audit-ingestor/.../kafka/AuditProducer.java` | +| **Topic create + `createPartitions`** | **Exists** | `audit-common/.../AuditMessageQueueUtils.java` | +| **XML partition config** | **Exists** | `audit-ingestor/.../conf/ranger-audit-ingestor-site.xml` | +| **`partition.plan.dynamic.enabled`** etc. | **Proposed** | Not in `AuditServerConstants.java` or site XML yet | + +--- + +### Flow 2 in code today (plugin POST → Kafka) + +**Description:** Every plugin audit hits the same Java call chain today. Partition key is `agentId` (plugin id). `AuditPartitioner` picks the Kafka partition from **static XML** at startup (not from plan topic yet). + +| Step | Class / file | +|------|----------------| +| 1 | `AuditREST.logAccessAudit()` — `audit-ingestor/.../rest/AuditREST.java` | +| 2 | `AuditDestinationMgr.logBatch()` — `audit-ingestor/.../producer/AuditDestinationMgr.java` | +| 3 | `AuditMessageQueue.log(batch)` — `audit-ingestor/.../kafka/AuditMessageQueue.java` | +| 4 | `AuditProducer.sendBatch()` — `audit-ingestor/.../kafka/AuditProducer.java` | +| 5 | `KafkaProducer` key=`agentId` → `AuditPartitioner.partition()` | +| 6 | Topic **`ranger_audits`** | + +```mermaid +flowchart LR + Plugin[Ranger plugin] -->|POST /audit/access| REST[AuditREST] + REST --> Mgr[AuditDestinationMgr] + Mgr --> Queue[AuditMessageQueue] + Queue -->|key = agentId| Prod[AuditProducer] + Prod --> Part[AuditPartitioner] + Part --> Topic[(ranger_audits)] +``` + +**Partition key** in `AuditMessageQueue.java` — `authzEvent.getAgentId()`. + +**Custom partitioner** wired in `AuditProducer.java` when `configured.plugins` is set → `AuditPartitioner`. + +--- + +### Static partitioning today (`AuditPartitioner` = future bootstrap v1 logic) + +At startup, `configure()` reads the same XML properties the design uses for **first-pod bootstrap**. It builds **contiguous ranges** (not explicit JSON lists yet): + +```56:102:audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java + public void configure(Map configs) { + String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; + + String pluginsStr = getConfig(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); + configuredPlugins = pluginsStr.split(","); + // ... + defaultPartitionsPerPlugin = getIntConfig(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, ...); + // ... + for (int i = 0; i < configuredPlugins.length; i++) { + String overrideKey = propPrefix + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin; + int partitionCount = getIntConfig(configs, overrideKey, defaultPartitionsPerPlugin); + pluginPartitionCounts[i] = partitionCount; + } + // contiguous ranges → configuredPluginPartitionStart/End, bufferPartitionStart/Count +``` + +**Runtime routing** (round-robin for configured plugin, hash for buffer): + +```117:151:audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java + public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { + // ... + int pluginIndex = indexOfConfiguredPlugin(appId); + if (pluginIndex >= 0) { + // round-robin within plugin range + return start + subPartition; + } else { + // Unconfigured plugin - use buffer partitions + int p = Math.abs(appId.hashCode() % count) + start; + return Math.min(p, numPartitions - 1); + } + } +``` + +**Dynamic design change:** replace fixed `int[]` ranges with `AtomicReference` and explicit `[0,1,2,...]` lists from Kafka. + +--- + +### Topic size + `AdminClient.createPartitions` (reuse for REST POST/PATCH) + +Topic creation and partition **increase** already exist — the proposed `PartitionPlanService` should call the same patterns: + +```49:116:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java + public static String createAuditsTopicIfNotExists(Properties props, String propPrefix) { + // ... + int partitions = getPartitions(props, propPrefix); + // AdminClient.createTopics or updateExistingTopicPartitions +``` + +```230:249:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java + if (partitions > currentPartitions) { + // ... + newPartitionsMap.put(topicName, NewPartitions.increaseTo(partitions)); + CreatePartitionsResult createPartitionsResult = admin.createPartitions(newPartitionsMap); + createPartitionsResult.all().get(); +``` + +**Partition count from XML** (same inputs as bootstrap v1 JSON): + +```330:365:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java + private static int getPartitions(Properties prop, String propPrefix) { + String configuredPlugins = MiscUtil.getStringProperty(prop, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, ...); + if (configuredPlugins == null || configuredPlugins.trim().isEmpty()) { + totalPartitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, ...); + } else { + for (String plugin : configuredPlugins.split(",")) { + int partitionCount = MiscUtil.getIntProperty(prop, overrideKey, defaultPartitionsPerPlugin); + totalPartitions += partitionCount; + } + totalPartitions += bufferPartitions; + } + return totalPartitions; + } +``` + +--- + +### REST today vs proposed admin API + +**Exists** — plugin audit ingestion only: + +| Endpoint | File | +|----------|------| +| `GET /audit/health` | `AuditREST.java` | +| `GET /audit/status` | `AuditREST.java` | +| `POST /audit/access` | `AuditREST.java` — auth via `allowed.users` per service | + +**Proposed** — not in code yet: + +| Endpoint | Intended location | +|----------|-------------------| +| `GET /api/audit/partition-plan` | `AuditREST.java` | +| `POST /api/audit/partition-plan/plugins` | `PartitionPlanService.onboardPlugin` | +| `PATCH /api/audit/partition-plan/plugins/{pluginId}` | `PartitionPlanService.updatePlugin` | + +--- + +### Configuration constants today (`AuditServerConstants`) + +Properties used for **legacy / bootstrap** partitioning (no `partition.plan.*` yet): + +```56:85:audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java + public static final String PROP_TOPIC_PARTITIONS = "kafka.topic.partitions"; + public static final String PROP_PARTITIONER_CLASS = "kafka.partitioner.class"; + public static final String PROP_CONFIGURED_PLUGINS = "kafka.configured.plugins"; + public static final String PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN = "kafka.topic.partitions.per.configured.plugin"; + public static final String PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX = "kafka.plugin.partition.overrides."; + public static final String PROP_BUFFER_PARTITIONS = "kafka.topic.partitions.buffer"; + // ... + public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3; + public static final int DEFAULT_BUFFER_PARTITIONS = 9; +``` + +**Sample XML** — `audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml`: + +- `ranger.audit.ingestor.kafka.configured.plugins` +- `ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin` +- `ranger.audit.ingestor.kafka.topic.partitions.buffer` +- `ranger.audit.ingestor.kafka.plugin.partition.overrides.*` + +--- + +### Where new code plugs in + +**Description:** Solid boxes = **exists today**. Dashed = **proposed**. Audit hot path unchanged; dynamic plan adds watcher + admin REST + plan topic. + +```mermaid +flowchart TB + subgraph exists [Exists today] + Plugin[Plugin POST /access] + RESTA[AuditREST /access] + Queue[AuditMessageQueue] + Prod[AuditProducer] + Part[AuditPartitioner static XML] + Audits[(ranger_audits)] + Plugin --> RESTA --> Queue --> Prod --> Part --> Audits + end + + subgraph proposed [Proposed] + RESTP[AuditREST /partition-plan] + Svc[PartitionPlanService] + Watcher[PartitionPlanWatcher] + Plan[(ranger_audit_partition_plan)] + Admin[Admin / automation] + Admin --> RESTP --> Svc + Svc --> Plan + Svc -->|createPartitions| Audits + Watcher -->|read| Plan + Watcher -->|update memory| Part + end +``` + +--- + +## Core components (new code) + +| Component | Responsibility | +|-----------|----------------| +| **`PartitionPlan`** | Immutable snapshot: plugin → explicit partition ID list, buffer list, `version`, `topicPartitionCount`, `updatedAt` | +| **`PartitionPlanAllocator`** | Append-only rules: assign new tail partitions; never remove/reassign existing plugin partitions | +| **`PartitionPlanRegistry`** | Read/write plan to Kafka compacted topic | +| **`PartitionPlanService`** | Orchestrates GET/POST/PATCH: validate, allocate, AdminClient, registry write | +| **`PartitionPlanWatcher`** | Background thread on **every** ingestor pod; refreshes in-memory plan | +| **`AuditPartitioner` (modified)** | Route by plan: configured plugin → round-robin within its list; unknown → buffer list | +| **`AuditREST` (new endpoints)** | Admin API; AuthZ like `/access` | + +### Existing code to reuse (already in repo) + +**Description:** Build new partition-plan components on these classes — do not rewrite topic admin or produce plumbing. + +| Reuse | Path | Use for | +|-------|------|---------| +| `AuditMessageQueueUtils.createAuditsTopicIfNotExists()` | `audit-common/.../AuditMessageQueueUtils.java` | Pattern for `createPlanTopicIfNotExists()` | +| `AuditMessageQueueUtils.updateExistingTopicPartitions()` | same | REST POST/PATCH → grow `ranger_audits` | +| `AuditProducer` | `audit-ingestor/.../kafka/AuditProducer.java` | Existing Kafka producer; plan topic produce | +| `AuditPartitioner` | `audit-ingestor/.../kafka/AuditPartitioner.java` | Add `AtomicReference` | +| `AuditREST` + `isAllowedServiceUser()` | `audit-ingestor/.../rest/AuditREST.java` | Admin AuthZ pattern for `/partition-plan` | + +```mermaid +flowchart LR + subgraph new [Proposed new code] + PP[PartitionPlan] + Reg[PartitionPlanRegistry] + Svc[PartitionPlanService] + Watch[PartitionPlanWatcher] + end + + subgraph reuse [Reuse from repo] + Utils[AuditMessageQueueUtils] + Part[AuditPartitioner] + REST[AuditREST] + Prod[AuditProducer] + end + + Svc --> Utils + Svc --> Reg + Svc --> Prod + Watch --> Reg + Watch --> Part + REST --> Svc +``` + +--- + +## Partition plan topic (`ranger_audit_partition_plan`) + +Why a **separate** topic from `ranger_audits`: + +- Audit topic carries high-volume events; plan is low-volume config. +- Compacted config topic gives durable “latest value per key” semantics (like Kafka’s own `__consumer_offsets` pattern). + +Suggested topic config: + +```properties +partitions=1 +cleanup.policy=compact +min.compaction.lag.ms=0 # optional: faster visibility of new plan +``` + +**Key:** `ranger_audits` (audit topic name — allows multiple audit topics in future) +**Value:** JSON `PartitionPlan` + +When `dynamic.enabled=true`, each ingestor pod ensures this topic exists at startup via **`createPartitionPlanTopicIfNotExists()`** (implemented in `AuditMessageQueueUtils`; same AdminClient pattern as `createAuditsTopicIfNotExists()` for `ranger_audits`). If the topic is missing, the pod creates it with **1 partition** and **`cleanup.policy=compact`**. + +--- + +## Compacted topic semantics: append, retention, and performance + +This section documents how plan **writes** and **reads** behave on `ranger_audit_partition_plan` — use it when reviewing or extending the design (Phase 2 registry, Phase 3 watcher). + +### Does `writePlan` update the same record? + +**No — Kafka does not update records in place.** Each `writePlan` (REST POST onboard / PATCH update, bootstrap publish) **appends a new record** to partition 0: + +```text +offset 10: key=ranger_audits value=plan v1 +offset 11: key=ranger_audits value=plan v2 +offset 12: key=ranger_audits value=plan v3 +``` + +| Concept | Meaning | +|---------|---------| +| **Physical write** | Always **append** (new offset) | +| **Logical update** | Same **key** (`ranger_audits`) + compaction → only **latest value per key** matters | +| **`version` in JSON** | Application-level optimistic locking (REST `expectedVersion`); not a Kafka in-place edit | + +With `cleanup.policy=compact`, the log cleaner eventually **removes older records for the same key** and retains the **latest** value. After compaction, you typically have **~one record per audit topic key**, not an ever-growing history. + +### Phase 2 read strategy (`KafkaPartitionPlanRegistry.readPlan`) + +Current implementation (admin/bootstrap reads — **not** the audit hot path): + +1. Assign partition 0, `seekToBeginning`. +2. Poll until empty; for each record with matching key, parse JSON into `latest`. +3. Return **last matching record** (highest offset) — correct even if compaction has not run yet. + +This is intentionally simple for **rare** reads (startup, REST GET with force-read, compare-and-swap before POST/PATCH produce). + +### Will growing records hurt performance? + +**Not in normal Ranger use.** This topic is unlike `ranger_audits`: + +| Dimension | `ranger_audit_partition_plan` | `ranger_audits` | +|-----------|-------------------------------|-----------------| +| Write rate | Rare (ops: bootstrap, onboard, update) | Very high (every audit) | +| Typical keys | **1** (`ranger_audits`) | N/A (data topic) | +| Records retained (after compaction) | **~1 per key** | Full retention window | +| Audit POST hot path | **Does not read** this topic | Produces every audit | + +**Cost of `readPlan` full scan** ≈ number of records still on disk for that key before compaction: + +```text +~100 uncompacted versions × ~2 KB JSON ≈ 200 KB → usually well under 100 ms +``` + +Audit ingest throughput is **unaffected** — `AuditPartitioner` reads an in-memory plan only. + +### When record growth could matter (edge cases) + +| Situation | Risk | Mitigation | +|-----------|------|------------| +| Automation hammers PATCH/POST mutations (many versions per minute) | Full scan reads more uncompacted records | Rate-limit ops; rely on compaction; Phase 3 watcher uses **incremental** consume | +| Compaction lag / misconfigured topic | Old versions accumulate on disk | Monitor compaction; set `min.compaction.lag.ms` if needed; verify `cleanup.policy=compact` | +| Many independent audit topic keys on one partition | More keys × versions before compact | Unusual today; one key per deployment is the default design | +| Using full `seekToBeginning` on every watcher poll (anti-pattern) | Unnecessary IO every 30s | Phase 3: consume **new offsets only** after initial load | + +### Phase 3 watcher (planned — do not full-scan every refresh) + +| Phase | Read pattern | When | +|-------|--------------|------| +| **Phase 2** (`readPlan`) | Full compacted log scan | Rare: bootstrap, REST, admin debug | +| **Phase 3** (`PartitionPlanWatcher`) | Initial load once; then **incremental** consumer poll | Every `refresh.interval.ms` (default 30s) | + +Watcher should **not** repeat `seekToBeginning` on every refresh. After startup, track consumer offset and apply only **new** plan records. + +### Ops guardrails (design assumptions) + +- Plan changes are **infrequent** (human or CI), not per-request. +- Default deployment: **one** compacted key (`ranger_audits`) on **one** partition. +- Runtime routing uses **memory**; Kafka is control plane only. +- If plan update frequency ever approaches “every few seconds,” revisit watcher incremental consume and compaction tuning — not audit producer tuning. + +**Implementation reference:** `KafkaPartitionPlanRegistry` in `audit-ingestor/.../partition/` (Phase 2). See [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) Phase 2–3. + +--- + +## Partition plan JSON (canonical compacted-topic value) + +The compacted topic stores **one JSON document per audit topic key** — nothing else. This is the **only** durable source of truth when `dynamic.enabled=true`. + +**Key:** `ranger_audits` (audit topic name) +**Value:** JSON `PartitionPlan` (fields below — no extra metadata required): + +```json +{ + "topic": "ranger_audits", + "version": 5, + "topicPartitionCount": 28, + "updatedAt": "2026-06-02T12:00:00Z", + "updatedBy": "admin@example.com", + "plugins": { + "hdfs": { "partitions": [0, 1, 2, 3] }, + "hiveServer2": { "partitions": [4, 5, 6, 7, 8, 9] }, + "trino": { "partitions": [10, 11, 12, 13, 14, 15, 16, 17, 18] } + }, + "buffer": { + "partitions": [19, 20, 21, 22, 23, 24, 25, 26, 27] + } +} +``` + +| Field | Purpose | +|-------|---------| +| `topic` | Audit topic this plan applies to (matches compacted topic **key**) | +| `version` | Monotonic integer; optimistic locking on REST POST/PATCH mutations | +| `topicPartitionCount` | Must match Kafka partition count for `topic` | +| `updatedAt` / `updatedBy` | Audit trail (optional but recommended) | +| `plugins` | Map plugin id (`agentId`) → explicit partition ID list | +| `buffer` | Partition IDs for unknown / unconfigured plugins | + +**Create or update:** only via **REST** (`POST .../plugins` / `PATCH .../plugins/{pluginId}`) or **one-time bootstrap publish** when the topic is empty (see below). Producers and audit POST handlers **never** write to this topic. + +**Routing rules in `AuditPartitioner`:** + +1. Lookup `agentId` in `plan.plugins`. +2. If found → round-robin across that plugin’s partition list. +3. If not found → hash into `plan.buffer.partitions`. +4. Validate all partition IDs exist in `cluster.partitionsForTopic(topic)` before applying plan. + +--- + +## Bootstrap and multi-pod startup + +When `dynamic.enabled=true`, **Kafka compacted topic wins** over XML whenever a plan message already exists. + +### First pod: XML populates the plan topic + +On the **first ingestor startup** with dynamic mode enabled, if `ranger_audit_partition_plan` has **no message** for key `ranger_audits`, the first pod **builds v1 from XML and publishes it** to the compacted topic. That is how the plan registry is initially filled. + +#### When bootstrap-from-XML runs + +| Condition | Bootstrap from XML → plan topic? | +|-----------|----------------------------------| +| `dynamic.enabled=true` **and** plan topic empty | **Yes** — first pod publishes v1 | +| `dynamic.enabled=true` **and** plan already in Kafka | **No** — read Kafka only | +| `dynamic.enabled=false` or property absent | **No** — legacy XML at startup; plan topic unused | + +#### XML properties used to build v1 + +| XML property | Role in v1 plan | +|--------------|-----------------| +| `ranger.audit.ingestor.kafka.configured.plugins` | Plugin ids → keys in `plan.plugins` (order defines initial contiguous ranges) | +| `ranger.audit.ingestor.kafka.plugin.partition.overrides.` | Partition count for that plugin (length of its `partitions` list) | +| `ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin` | Default count per plugin when no override (typically **3**) | +| `ranger.audit.ingestor.kafka.topic.partitions.buffer` | Size of `plan.buffer.partitions` (typically **9**) | + +The bootstrap step **converts** today’s contiguous-range layout into **explicit partition ID lists** in JSON (same shape as REST-managed plans). + +#### Example: XML → v1 in `ranger_audit_partition_plan` + +**XML (simplified):** `configured.plugins=hdfs,hiveServer2`, default 3 partitions each, buffer 9. + +**Kafka compacted topic** — key `ranger_audits`, value: + +```json +{ + "topic": "ranger_audits", + "version": 1, + "topicPartitionCount": 15, + "updatedAt": "2026-06-02T10:00:00Z", + "updatedBy": "bootstrap", + "plugins": { + "hdfs": { "partitions": [0, 1, 2] }, + "hiveServer2": { "partitions": [3, 4, 5] } + }, + "buffer": { + "partitions": [6, 7, 8, 9, 10, 11, 12, 13, 14] + } +} +``` + +After this one-time publish, **all routing and later changes** use the plan in Kafka (REST onboard/update), not XML edits. + +```mermaid +flowchart LR + XML[ranger-audit-ingestor-site.xml] -->|first pod only| Build[Build v1] + Build --> Publish[Publish to ranger_audit_partition_plan] + Publish --> Mem[Load into memory] + Mem --> Audits[Serve audits] +``` + +--- + +### Pod 1 (first ingestor, empty plan topic) + +**Description:** First pod with `dynamic.enabled=true` creates plan topic (if needed), seeds **v1 from XML** into Kafka, loads into memory. XML is used **once**. + +| Step | Action | +|------|--------| +| 1 | `PartitionPlanWatcher` starts | +| 2 | `createPlanTopicIfNotExists()` — idempotent ([Race A](#race-a--multiple-pods-create-the-plan-topic-at-the-same-time)) | +| 3 | Read plan topic → no message for key `ranger_audits` | +| 4 | Build v1 from XML → explicit partition lists | +| 5 | Publish v1 to `ranger_audit_partition_plan` ([Race B](#race-b--multiple-pods-publish-plan-v1-when-topic-exists-but-has-no-message)) | +| 6 | Re-read compacted topic → install v1 in memory | +| 7 | Serve audits (memory only on hot path) | + +```mermaid +flowchart TD + Start[Pod 1 starts] --> Watcher[PartitionPlanWatcher] + Watcher --> Create[createPlanTopicIfNotExists] + Create --> Read{Plan message exists?} + Read -->|No| XML[Build v1 from XML] + XML --> Publish[Publish v1 to plan topic] + Publish --> Reread[Re-read compacted topic] + Reread --> Mem[Install in memory] + Mem --> Audits[Serve POST /access] +``` + +**Alternative:** wait for first admin POST onboard instead of auto-publish — possible, but auto-publish is recommended for pod 2+. + +### Pod 2+ (later ingestors, plan already in Kafka) + +**Description:** Plan already in Kafka — **read only**, never re-bootstrap from XML for routing. + +| Step | Action | +|------|--------| +| 1 | `PartitionPlanWatcher` starts | +| 2 | `createPlanTopicIfNotExists()` — usually no-op (topic exists) | +| 3 | Read plan version **N** from compacted topic | +| 4 | Install into memory — **ignore XML** | +| 5 | Serve audits | + +```mermaid +flowchart TD + Start[Pod 2+ starts] --> Watcher[PartitionPlanWatcher] + Watcher --> Create[createPlanTopicIfNotExists no-op] + Create --> Read[Read plan vN from Kafka] + Read --> Mem[Install in memory] + Mem --> Audits[Serve POST /access] + XML[XML config] -.->|not used for routing| Mem +``` + +**Do not** bootstrap-from-XML again if a plan message exists — avoids ConfigMap drift across pods. + +--- + +### Race A — multiple pods create the plan **topic** at the same time + +Several ingestor pods can start together when `ranger_audit_partition_plan` **does not exist yet**. Each pod may call `listTopics()`, see nothing, and call `createTopics()` — same class of race as `ranger_audits` creation in `AuditMessageQueueUtils.createAuditsTopicIfNotExists()`. + +**Required behavior (idempotent topic create):** + +**Description:** Multiple pods may call `createTopics()` at once. **Already exists = success.** Only fail on real errors (ACL, Kafka down). + +| Step | Action | +|------|--------| +| 1 | `listTopics()` — if topic exists → OK, continue | +| 2 | `createTopics(1 partition, cleanup.policy=compact)` | +| 3 | Success → `waitUntilTopicReady` → continue | +| 4 | Failure **already exists** → treat as success, `describe` topic, continue | +| 5 | Other failure → fail startup with clear error | + +```mermaid +flowchart TD + Start[createPlanTopicIfNotExists] --> List{Topic exists?} + List -->|Yes| OK[Return OK] + List -->|No| Create[createTopics] + Create --> Success{Result?} + Success -->|OK| Wait[waitUntilTopicReady] + Wait --> OK + Success -->|Already exists| OK + Success -->|Other error| Fail[Fail startup] +``` + +| Outcome | Pod behavior | +|---------|----------------| +| This pod creates topic | Log info, proceed to read/publish plan | +| Peer created topic first | Catch already-exists, describe topic, proceed | +| Create fails for other reason | Fail startup (Kafka unreachable, ACL denied) | + +```mermaid +sequenceDiagram + participant PodA as ingestor pod A + participant PodB as ingestor pod B + participant Kafka as Kafka Admin + + PodA->>Kafka: listTopics — plan topic missing + PodB->>Kafka: listTopics — plan topic missing + PodA->>Kafka: createTopics(ranger_audit_partition_plan) + PodB->>Kafka: createTopics(ranger_audit_partition_plan) + Kafka-->>PodA: OK + Kafka-->>PodB: already exists + PodB->>PodB: treat as success, continue + Note over PodA,PodB: both proceed to plan read / bootstrap +``` + +**Implementation note:** reuse or factor shared logic from `audit-common/.../AuditMessageQueueUtils.java` (`createAuditsTopicIfNotExists`) and add explicit **already-exists** handling if not present today (concurrent `createTopics` on `ranger_audits` has the same gap). + +--- + +### Race B — multiple pods publish plan **v1** when topic exists but has no message + +**Description:** Topic exists but no plan message yet. Both pods may build v1 — **re-read before/after produce**; mandatory read-back; adopt Kafka’s plan. + +| Step | Action | +|------|--------| +| 1 | Both pods: `ensurePlanTopicExists()` (Race A done) | +| 2 | Read plan → null | +| 3 | Build v1 from XML | +| 4 | **Re-read** — if v1 appeared → install, skip publish | +| 5 | Produce local v1 (first wins on offset) | +| 6 | **Mandatory re-read** → install stored plan (not local object only) | + +```mermaid +flowchart TD + Start[bootstrapPlanIfEmpty] --> Ensure[Race A: topic exists] + Ensure --> Read1[Read plan from Kafka] + Read1 -->|plan exists| Install1[Install and return] + Read1 -->|empty| Build[Build v1 from XML] + Build --> Read2[Re-read plan] + Read2 -->|peer published| Install2[Install peer plan return] + Read2 -->|still empty| Produce[Produce local v1] + Produce --> Read3[Mandatory read-back] + Read3 --> Install3[Install Kafka plan] +``` + +```mermaid +sequenceDiagram + participant A as pod A + participant B as pod B + participant Plan as ranger_audit_partition_plan + + A->>Plan: read → empty + B->>Plan: read → empty + A->>Plan: produce v1 + B->>Plan: produce v1 or re-read sees v1 + A->>Plan: read-back → install v1 + B->>Plan: read-back → install same v1 + Note over A,B: both pods same routing +``` + +This preserves backward compatibility with `ranger-audit-ingestor-site.xml` for greenfield installs. + +--- + +## Kafka read load (REST + background thread only) + +The plan topic is **low volume** (config, not audit events). Avoid reading it on the hot audit path. + +| Code path | Reads plan topic? | Notes | +|-----------|-------------------|--------| +| Plugin audit POST → `AuditMessageQueue` → `AuditPartitioner.partition()` | **No** | Reads `AtomicReference` in memory only | +| `PartitionPlanWatcher` (background thread) | **Yes** | Poll / consume on interval or on compacted message | +| REST `GET /api/audit/partition-plan` | **Yes** | Admin read; may return cached in-memory plan if fresh | +| REST POST/PATCH plugins | **Yes** | Read current version, then produce new version | +| Audit produce to `ranger_audits` | **No** | Unrelated topic | + +**Design rule:** at most **one background consumer per ingestor pod** plus **occasional REST reads/writes**. No per-request Kafka access for partition routing. + +Recommended watcher behavior: + +- **Startup:** one read to load initial plan (from compacted topic, or bootstrap-then-read). +- **Runtime:** consumer poll with long interval (`refresh.interval.ms`, default 30s) **or** event-driven on new compacted message — not tight spin loops. +- **On failure:** keep **last known good** plan in memory; do not block audit POST. + +With a single-partition compacted topic and ~30s refresh, Kafka load from N ingestor pods is negligible compared to `ranger_audits` traffic. + +**Append vs update:** each plan write adds a record; compaction + “latest per key” semantics mean the effective store is one current plan. See [Compacted topic semantics: append, retention, and performance](#compacted-topic-semantics-append-retention-and-performance). + +--- + +## REST API (control plane) + +The partition-plan admin API exposes **three endpoints**. All mutation requests use optimistic locking via `expectedVersion` (current plan version from `GET`). Reject with `409 Conflict` when stale; response body includes the current plan. + +**Auth:** Kerberos/JWT + admin role (not the same as plugin audit POST users). When `kafka.partition.plan.allowed.users` is configured, only those short names may call these endpoints. + +### GET `/api/audit/partition-plan` + +Returns the current plan from the in-memory holder (same JSON shape as the compacted Kafka value). + +### POST `/api/audit/partition-plan/plugins` — onboard plugin + +Onboards a plugin from the buffer to dedicated partitions and registers service allowlists in **one** plan version bump. + +**Required fields:** `pluginId`, `partitionCount`, `expectedVersion`, `services` + +**`services` map:** repo name → `{ "allowedUsers": [...] }`. Must be non-empty. Each entry is tagged with `pluginId` in the stored plan. + +```json +{ + "pluginId": "hiveServer2", + "partitionCount": 3, + "expectedVersion": 1, + "services": { + "dev_hive": { "allowedUsers": ["hive"] }, + "dev_hive2": { "allowedUsers": ["hive2"] } + } +} +``` + +Server: takes partition IDs from buffer (or grows `ranger_audits` tail), adds plugin assignment, merges services with `pluginId`, writes plan v(N+1). + +### PATCH `/api/audit/partition-plan/plugins/{pluginId}` — update plugin + +Updates an onboarded plugin: scale tail partitions and/or mutate service allowlists scoped to `{pluginId}` in one version bump. + +**Required:** `expectedVersion` + +**At least one of:** + +| Field | Purpose | +|-------|---------| +| `additionalPartitions` | int ≥ 1 — append tail partition IDs (same append-only semantics as legacy scale) | +| `addServices` | map repo → `{ "allowedUsers": [...] }` — add repos tagged with path `pluginId` | +| `updateServices` | map repo → `{ "allowedUsers": [...] }` — replace allowlist for repos owned by path `pluginId` (or legacy entries with no `pluginId`) | +| `removeServices` | list of repo names — remove allowlist entries owned by path `pluginId` (or legacy entries with no `pluginId`) | + +**Example — Hive multi-repo lifecycle** (onboard two repos, add a third, remove one, scale): + +```bash +# 1) Onboard hiveServer2 with dev_hive + dev_hive2 (plan v1 → v2) +curl -X POST .../api/audit/partition-plan/plugins -d '{ + "pluginId": "hiveServer2", + "partitionCount": 3, + "expectedVersion": 1, + "services": { + "dev_hive": { "allowedUsers": ["hive"] }, + "dev_hive2": { "allowedUsers": ["hive2"] } + } +}' + +# 2) Add dev_hive3, remove dev_hive2 (plan v2 → v3) +curl -X PATCH .../api/audit/partition-plan/plugins/hiveServer2 -d '{ + "expectedVersion": 2, + "addServices": { "dev_hive3": { "allowedUsers": ["hive3"] } }, + "removeServices": ["dev_hive2"] +}' + +# 3) Scale +2 tail partitions (plan v3 → v4) +curl -X PATCH .../api/audit/partition-plan/plugins/hiveServer2 -d '{ + "expectedVersion": 3, + "additionalPartitions": 2 +}' +``` + +Stored service entries include optional `"pluginId": "hiveServer2"` for ownership tracking. + +**Removed endpoints** (consolidated above): `PATCH /api/audit/partition-plan`, `POST /api/audit/partition-plan/services`, and separate promote-only / scale-only semantics. + +> **Future work (not in this change):** bootstrap **v0** split — seeding `services` from XML separately from plugin partition assignments. Current bootstrap still publishes a single v1 plan from XML as today. + +--- + +## Mutation handler flow (POST onboard / PATCH update) + +**Description:** Admin POST/PATCH on any ingestor pod. Grow `ranger_audits` **before** publishing plan that references new partition IDs. Server assigns `version = N+1`; client sends `expectedVersion` only. + +| Step | Action | +|------|--------| +| 1 | Authenticate + authorize admin | +| 2 | Load plan from compacted topic (version **N**) | +| 3 | If `expectedVersion != N` → **409** + current plan | +| 4 | Validate + allocate (onboard: partitions + services; update: scale and/or service mutations) | +| 5 | If needed → `AdminClient.createPartitions(ranger_audits)` | +| 6 | **Re-read** plan — still **N**? else **409** | +| 7 | Produce plan **N+1** to plan topic | +| 8 | **Read-back** — matches intent? else **409** | +| 9 | Return **200** + full plan | + +```mermaid +flowchart TD + Admin[Admin POST/PATCH] --> Auth[Auth + authorize] + Auth --> Load[Load plan vN] + Load --> Check1{expectedVersion = N?} + Check1 -->|No| R409a[409 + current plan] + Check1 -->|Yes| Valid[Validate + allocate] + Valid --> Grow{Need more partitions?} + Grow -->|Yes| CP[createPartitions ranger_audits] + Grow -->|No| Reread[Re-read plan] + CP --> Reread + Reread --> Check2{Still vN?} + Check2 -->|No| R409b[409 peer won] + Check2 -->|Yes| Produce[Produce vN+1] + Produce --> Back[Read-back verify] + Back --> Check3{Matches intent?} + Check3 -->|No| R409c[409 lost race] + Check3 -->|Yes| OK[200 + plan] +``` + +**Order matters:** increase audit topic **before** plan references new IDs. + +**Version is server-assigned** — clients must not set the new version number. + +--- + +## Concurrent updates from multiple ingestor pods + +REST is exposed on **every** ingestor pod behind a load balancer. Two admins (or automation jobs) can hit **different pods at the same time**. Writes must not double-allocate partition IDs or overwrite each other silently. + +### Recommended approach: optimistic concurrency + compare-and-swap + +This is the **default** implementation — no leader election required. + +| Mechanism | Role | +|-----------|------| +| **`expectedVersion` in request** | Client declares which plan it based its change on | +| **Early reject** | If compacted topic already at version ≠ `expectedVersion` → **409 Conflict** before allocation | +| **Re-read before produce** | After `createPartitions` (if any), re-read topic; if version changed → **409** without producing | +| **Single-partition plan topic** | All plan writes are totally ordered by Kafka offset | +| **Read-back after produce** | Confirm the compacted value is the plan this handler intended; else **409** | +| **Client retry** | On 409: `GET` latest plan, re-apply intent with new `expectedVersion` | + +```mermaid +sequenceDiagram + participant AdminA as Admin / job A + participant PodA as ingestor pod A + participant PodB as ingestor pod B + participant PlanTopic as ranger_audit_partition_plan + + Note over PlanTopic: current version = 4 + + AdminA->>PodA: PATCH .../plugins/hiveServer2 expectedVersion=4 + AdminA->>PodB: POST .../plugins expectedVersion=4 + + PodA->>PlanTopic: read plan → v4 + PodB->>PlanTopic: read plan → v4 + + PodA->>PodA: allocate + createPartitions if needed + PodB->>PodB: allocate + createPartitions if needed + + PodA->>PlanTopic: re-read → still v4 + PodB->>PlanTopic: re-read → still v4 + + PodA->>PlanTopic: produce v5 (update hiveServer2) + PlanTopic-->>PodB: compacted update visible + + PodB->>PlanTopic: re-read → now v5 + PodB-->>AdminA: 409 Conflict + current plan v5 + + Note over AdminA: retry with expectedVersion=5 + + AdminA->>PodB: POST .../plugins expectedVersion=5 + PodB->>PlanTopic: read v5 → allocate → produce v6 + PodB-->>AdminA: 200 OK plan v6 +``` + +### What each outcome means + +| HTTP | Meaning | Client action | +|------|---------|---------------| +| **200** | This handler’s plan v(N+1) is in the compacted topic | Done | +| **409 Conflict** | Stale `expectedVersion` or lost compare-and-swap race | `GET` plan, merge intent, retry with new `expectedVersion` | +| **503** | `createPartitions` failed or Kafka produce failed | Retry same request (idempotent if version unchanged) | + +**409 response body** should include the **current plan** (version, plugins, buffer) so callers can retry without a separate GET. + +### Why this is safe for partition allocation + +Two pods both scaling from v4 might temporarily compute overlapping tail IDs in memory — only one produce wins the compare-and-swap. The loser gets **409** and must **re-read v5** and allocate from the **new tail**, not reuse its stale allocation. Append-only validation on POST/PATCH rejects plans that steal IDs already assigned in the current version. + +### Residual race: both produce before either re-reads + +**Description:** Plan topic has **one partition** — Kafka orders writes by offset. Only one v5 survives in compaction; loser returns **409**, client retries on winning plan. + +| Step | What happens | +|------|----------------| +| 1 | Pod A and B both pass re-read at v4 | +| 2 | Both produce v5 — higher **offset wins** in compacted topic | +| 3 | Loser read-back ≠ intended plan → **409** to client (not 200) | +| 4 | All watchers load **single** winning v5 | + +```mermaid +sequenceDiagram + participant A as pod A handler + participant B as pod B handler + participant Plan as plan topic 1 partition + + Note over Plan: both saw v4 at re-read + A->>Plan: produce v5 offset 100 + B->>Plan: produce v5 offset 101 + Note over Plan: compaction keeps offset 101 + A->>Plan: read-back + A-->>A: content != intent → 409 + B->>Plan: read-back + B-->>B: matches → 200 + Note over Plan: watchers all load one v5 +``` + +No silent split-brain — one plan version in registry; loser retries with `expectedVersion=5`. + +### Optional: stricter serialization (if you want zero client retry) + +| Approach | When to use | +|----------|-------------| +| **K8s Lease** (`partition-plan-writer`) | Only the lease holder executes PATCH/POST mutations; other pods return **503 Retry on leader** or proxy internally | +| **Single admin job / CI** | Human or pipeline serializes changes — sufficient for many ops teams | +| **Dedicated single-replica “admin” ingestor** | Extreme isolation; usually unnecessary | + +Start with **optimistic concurrency + compare-and-swap + client retry**. Add leader lease only if automation cannot tolerate 409 retries. + +### What ingestor pods do *not* do + +- **No cross-pod locking** on the audit POST path. +- **No per-pod plan writes** from the watcher (watchers are **read-only**). +- **No merge of concurrent admin intent on the server** — conflicts are explicit 409; the client or operator merges. + +--- + +## How every ingestor pod stays in sync + +**Description:** Each pod runs **`PartitionPlanWatcher`** (background only). `AuditPartitioner` reads memory — never plan topic on audit POST. + +| Option | Behavior | +|--------|----------| +| **A — Consumer (recommended)** | Group `ranger_audit_partition_plan_watcher`; on message → validate → `partitionPlanRef.set()` | +| **B — Periodic poll** | Every `refresh.interval.ms` (default 30s) read latest compacted value | + +```mermaid +flowchart TB + Plan[(ranger_audit_partition_plan)] + + subgraph pod1 [Ingestor pod 1] + W1[PartitionPlanWatcher] + M1[(Memory plan)] + P1[AuditPartitioner] + W1 --> M1 --> P1 + end + + subgraph pod2 [Ingestor pod 2] + W2[PartitionPlanWatcher] + M2[(Memory plan)] + P2[AuditPartitioner] + W2 --> M2 --> P2 + end + + Plan --> W1 + Plan --> W2 + + Plugin1[Plugin POST] --> P1 + Plugin2[Plugin POST] --> P2 +``` + +**Latency:** all pods converge within ~30s or on Kafka message. **No restart** when plan changes. + +--- + +## Pod crash / restart + +**Description:** Same as **Pod 2+** — memory lost on crash; plan survives in Kafka compacted topic. + +| Step | Action | +|------|--------| +| 1 | Pod dies — in-memory plan lost | +| 2 | New pod starts | +| 3 | Watcher reads plan **vN** from Kafka | +| 4 | Install memory — routing restored | + +```mermaid +flowchart LR + Crash[Pod crash] --> Lost[Memory lost] + Lost --> New[New pod starts] + New --> Read[Read plan vN from Kafka] + Read --> Mem[Install memory] + Mem --> OK[Same routing as other pods] +``` + +Plan is **not** on pod disk — Kafka compacted topic is durable store. + +--- + +## Worked example: hdfs + hiveServer2, then onboard trino + +**Initial plan (v1)** — from bootstrap: + +| Plugin | Partitions | +|--------|------------| +| hdfs | [0,1,2] | +| hiveServer2 | [3,4,5] | +| buffer | [6..14] (9 partitions) | + +Topic: **15** partitions. + +**Unknown plugin `trino` sends audits** → routes to buffer (hash among [6..14]). + +**Admin: POST .../plugins — onboard trino (partitionCount=3, mandatory services)** + +```json +{ + "pluginId": "trino", + "partitionCount": 3, + "expectedVersion": 1, + "services": { + "dev_trino": { "allowedUsers": ["trino"] } + } +} +``` + +1. Allocator takes IDs [6,7,8] from buffer (or adds tail if buffer policy prefers grow-first). +2. buffer becomes [9..14]; `dev_trino` allowlist merged with `pluginId: trino`. +3. Plan v2 published. +4. All ingestors pick up v2 within refresh interval. +5. **No ingestor restart.** + +**Later: PATCH .../plugins/hiveServer2 — additionalPartitions=3** + +1. Increase topic 15 → 18 if needed. +2. Append partition IDs [15,16,17] to hiveServer2 list (append-only — hdfs and trino lists unchanged). +3. Plan v3 published. + +This fixes the “change hdfs override reshuffles everyone” problem from static contiguous allocation. + +--- + +## Solr / HDFS dispatchers + +Dispatchers (`README-KAFKA-DISPATCHERS.md`) **ignore** the partition plan: + +- They subscribe to **all** partitions of `ranger_audits`. +- When audit topic grows 15 → 18, both consumer groups rebalance and assign new partitions. +- No dispatcher code or config change for plugin onboarding. + +You may need to scale dispatcher `thread.count` / replicas if partition count grows significantly. + +--- + +## Failure modes and handling + +| Failure | Behavior | +|---------|----------| +| Plan topic unreadable | Keep **last known good** plan in memory; log error; `/status` shows degraded | +| Invalid plan JSON | Reject write at REST; do not apply on read | +| AdminClient partition increase fails | Do **not** publish new plan; return 503 to caller | +| Partial write (partitions increased but plan not written) | Retry POST/PATCH idempotently; allocator sees higher topic count | +| Two concurrent POST/PATCH (different pods) | `expectedVersion` + re-read before produce + read-back → one **200**, one **409**; client retries on latest version | +| Plugin sends audits during plan swap | `AtomicReference` swap is atomic; brief mix of old/new routing acceptable for audits | + +--- + +## Feature flag: backward compatibility (old vs dynamic behavior) + +**Description:** Dynamic mode is **opt-in**. Property **missing** or **`false`** → legacy behavior (today). **`true`** → Kafka plan topic + watcher + REST. + +Dynamic mode is **disabled** when: + +- `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` is **`false`**, or +- the property is **missing / not set** (default = **false**) + +```mermaid +flowchart TD + Flag{dynamic.enabled?} + Flag -->|false or absent| Legacy[Legacy mode] + Flag -->|true| Dynamic[Dynamic mode] + + Legacy --> L1[AuditPartitioner from XML at startup] + Legacy --> L2[No plan topic / no watcher] + Legacy --> L3[XML change + restart] + + Dynamic --> D1[Plan from Kafka compacted topic] + Dynamic --> D2[Watcher + REST enabled] + Dynamic --> D3[XML bootstrap once if empty] +``` + +### When dynamic mode is OFF (default — today’s behavior) + +| Area | Behavior | +|------|----------| +| Partition mapping | `AuditPartitioner.configure()` at startup from XML: `configured.plugins`, overrides, contiguous ranges + buffer | +| Topic partitions | Created/updated via `AuditMessageQueueUtils.getPartitions()` from XML (sum + buffer) | +| Changes | Update XML → restart audit-ingestor | +| REST `/api/audit/partition-plan` | **Not registered** or returns `404` / `503 Feature disabled` | +| `PartitionPlanWatcher` | **Not started** | +| Plan topic `ranger_audit_partition_plan` | **Not used** (may be omitted from cluster) | + +This preserves **100% backward compatibility** for existing deployments with no config change. + +### When dynamic mode is ON (`dynamic.enabled=true`) + +| Area | Behavior | +|------|----------| +| Partition mapping | Runtime `PartitionPlan` from Kafka compacted topic (XML seeds bootstrap only if plan topic empty) | +| Topic partitions | AdminClient increase driven by plan / REST | +| Changes | REST → Kafka plan topic; all pods refresh via watcher — **no restart** | +| REST partition-plan endpoints | **Enabled** (admin AuthZ) | +| `PartitionPlanWatcher` | **Started** on every ingestor pod | + +### Recommended default in code and sample XML + +```xml + + ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled + false + + Enable Kafka-backed dynamic partition plan (registry topic + REST + watcher). + Default false: use legacy XML-based AuditPartitioner behavior at startup. + If property is absent, treat as false. + + +``` + +Only set to `true` after `ranger_audit_partition_plan` topic exists and ops are ready to manage plans via REST. + +### Property defaults when `dynamic.enabled=true` + +If dynamic mode is on, missing optional properties use **built-in defaults** (do not fall back to legacy XML routing): + +| Property | If absent | Effective value | +|----------|-----------|-----------------| +| `partition.plan.topic` | Use default | **`ranger_audit_partition_plan`** | +| `partition.plan.refresh.interval.ms` | Use default | **`30000`** (30 seconds) | + +**Startup behavior with defaults:** + +**Description:** When `dynamic.enabled=true`, optional properties default as below. Fail startup if Kafka/plan topic unavailable — do not silently fall back to legacy XML. + +| Step | Action | +|------|--------| +| 1 | Resolve plan topic → **`ranger_audit_partition_plan`** (unless overridden) | +| 2 | **`createPlanTopicIfNotExists()`** — idempotent ([Race A](#race-a--multiple-pods-create-the-plan-topic-at-the-same-time)) | +| 3 | Start **`PartitionPlanWatcher`** | +| 4 | If no plan message → bootstrap v1 from XML ([Race B](#race-b--multiple-pods-publish-plan-v1-when-topic-exists-but-has-no-message)) | + +```mermaid +flowchart TD + Start[Ingestor startup dynamic=true] --> Topic[Resolve plan topic name] + Topic --> Create[createPlanTopicIfNotExists Race A] + Create --> Watcher[Start PartitionPlanWatcher] + Watcher --> Empty{Plan message empty?} + Empty -->|Yes| Boot[Bootstrap v1 from XML Race B] + Empty -->|No| Load[Load plan from Kafka] + Boot --> Ready[Ready serve audits] + Load --> Ready +``` + +**You do not need** to set `partition.plan.topic` unless using a non-default name. + +**Misconfiguration:** If `dynamic.enabled=true` but Kafka unreachable → **fail startup** (avoid split routing across pods). + +--- + +## What stays in XML (bootstrap only) + +| Property | Role after dynamic mode | +|----------|-------------------------| +| `configured.plugins` | Initial plan seed only (first pod bootstrap when plan topic empty) | +| `plugin.partition.overrides.*` | Initial counts when seeding v1 | +| `topic.partitions.buffer` | Initial buffer list size when seeding v1 | +| `topic.partitions.per.configured.plugin` | Default per-plugin count when seeding v1 (if no override) | +| `partitioner.class` | Still `AuditPartitioner` at JVM startup | +| `partition.plan.topic` | Compacted registry topic name; **default `ranger_audit_partition_plan`** if unset | +| `partition.plan.dynamic.enabled` | **`false` by default** — set `true` to enable dynamic mode | + +When `dynamic.enabled` is **false** or **unset**, XML properties above drive **legacy startup behavior** (not bootstrap-only). + +Runtime changes (when dynamic enabled) go through **REST → Kafka plan topic**, not XML edits. + +--- + +## Bootstrap configuration (new properties) + +```xml + + + ranger.audit.ingestor.kafka.partition.plan.topic + ranger_audit_partition_plan + + + ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms + 30000 + + + ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled + false + Default false. Set true to enable dynamic partition plan. If absent, use legacy XML behavior. + +``` + +Existing `configured.plugins` / overrides remain **bootstrap** when the compacted topic has no plan yet. + +--- + +## Implementation order (recommended) + +**Description:** Build bottom-up — model and registry first, then partitioner + watcher, then REST and ops. + +| Phase | Deliverable | +|-------|-------------| +| 1 | `PartitionPlan` model + allocator tests (append-only) | +| 2 | Registry read/write + `createPlanTopicIfNotExists` (Race A) | +| 3 | Plan-aware `AuditPartitioner` + watcher + bootstrap (Race B) | +| 4 | REST GET (inspect plan) | +| 5 | REST GET / POST / PATCH plugins | `PartitionPlanService` | +| 6 | AuthZ + `/status` (`plan.version`, watcher timestamp) | +| 7 | Ops runbook — REST only when dynamic=true → [README-KAFKA-PARTITION-PLAN-OPS-RUNBOOK.md](README-KAFKA-PARTITION-PLAN-OPS-RUNBOOK.md) | +| 8 | Migrate deployments — publish v1 from current XML | + +```mermaid +flowchart LR + P1[1 Model + allocator] --> P2[2 Registry + topic create] + P2 --> P3[3 Partitioner + watcher] + P3 --> P4[4 REST GET] + P4 --> P5[5 REST POST/PATCH plugins] + P5 --> P6[6 AuthZ + status] + P6 --> P7[7 Runbook] + P7 --> P8[8 Migration v1] +``` + +--- + +## Design rationale and tradeoffs + +**Assessment:** This is a **sound approach** for Ranger audit-ingestor: Kafka is already required, partition-plan changes are infrequent (ops events, not per-request), and the design keeps the audit hot path off Kafka. It is not the only valid design, but the tradeoffs fit the problem well. + +### Why this design works + +| Principle | How this design delivers | +|-----------|--------------------------| +| **Established pattern** | Kafka compacted topic as config store (same idea as Connect/Streams config topics): one key → latest plan, durable across restarts | +| **Hot path stays fast** | `AuditPartitioner.partition()` reads `AtomicReference` only; plan topic touched by watcher + REST, not every audit POST | +| **Append-only explicit lists** | Avoids reshuffling later plugins when an early plugin scales (main limitation of static contiguous XML allocation) | +| **Multi-pod consistency** | Pod 1 seeds Kafka from XML once; pod 2+ and restarts read Kafka; all replicas converge on the same JSON | +| **Safe rollout** | `dynamic.enabled` defaults to `false` — existing deployments unchanged | +| **Clean consumer boundary** | Solr/HDFS dispatchers consume all partitions; plan is producer-side only | + +### Acceptable tradeoffs + +| Concern | Severity | Mitigation in this design | +|---------|----------|---------------------------| +| Kafka required at startup when dynamic=true | Medium | Fail startup with clear error; do not silently fall back to legacy XML (would split routing across pods) | +| Two pods bootstrap race on empty plan topic | Low | **Race A:** idempotent `createPlanTopicIfNotExists` + already-exists OK; **Race B:** re-read before/after v1 produce | +| Watcher refresh lag (~30s default) | Low | Acceptable for plugin onboarding; tune `refresh.interval.ms` if needed | +| Ops edit XML expecting live effect | Medium | When dynamic=true, runbook: change plan via REST, not XML (XML is bootstrap-only) | +| `createPartitions` succeeds but plan write fails | Medium | Increase audit topic **before** publishing plan; retry POST/PATCH idempotently | +| ConfigMap drift between pods | Low | Kafka is source of truth once v1 exists | + +These are normal distributed-config caveats, not fundamental flaws. + +### When this approach is a poor fit + +Avoid dynamic mode (or reconsider the registry) if: + +- Plan changes happen **very frequently** (seconds) — watcher lag and consumer rebalance churn add cost. +- Kafka is **unavailable or not part of the stack** — use legacy XML or an external config store instead. +- You operate **many independent audit topics** with separate plans — still workable (one compacted key per topic) but REST and ops complexity grow. +- You require **strict per-plugin ordering across plan migrations** — append-only reduces reshuffle risk, but any partition reassignment still needs operational care. + +For typical Ranger deployments (infrequent plugin onboard/scale, Kafka already present), none of these apply strongly. + +### Alternatives considered + +| Option | Tradeoff vs this design | +|--------|-------------------------| +| **XML + restart only** (today) | Simplest; no dynamic onboarding without downtime | +| **Postgres / ZooKeeper** | Stronger CRUD and consistency; extra infra this design deliberately avoids | +| **K8s ConfigMap + watch** | K8s-centric; awkward for mixed environments, multi-cluster, or REST-driven automation | +| **Leader pod writes local XML** | Breaks with multiple replicas — rejected | + +Given **no new infra**, **multi-replica ingestor**, **REST-driven ops**, and **survive pod restart**, Kafka compacted topic + REST + background watcher is a strong fit. + +### Implementation guardrails + +1. **Bootstrap:** `createPlanTopicIfNotExists()` must be idempotent (Race A); after plan publish, always re-read compacted topic (Race B). +2. **GET `/partition-plan`:** prefer in-memory plan for normal reads; optional force-read from Kafka for debugging. +3. **`/status`:** expose `plan.version` and watcher last-updated timestamp. +4. **Mutation validation:** optional strict mode — reject writes that shrink or reassign existing plugin partition IDs. +5. **Runbook:** document that when `dynamic.enabled=true`, runtime changes go through REST, not XML edits. + +--- + +## Summary + +| Question | Answer | +|----------|--------| +| Postgres/ZK needed? | **No** — use Kafka compacted topic + AdminClient | +| Update XML in pod? | **No** for runtime — XML is bootstrap only | +| REST endpoint? | **Yes** — writes plan to Kafka + increases partitions via AdminClient | +| Pod crash/restart? | Re-read plan from Kafka compacted topic | +| Other pods notified? | All pods watch same topic / poll same plan | +| Dispatchers affected? | Rebalance only when partition count grows; no plan API needed | diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java index 5bffcadcdfc..54ca5480d42 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java @@ -24,12 +24,14 @@ import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -38,6 +40,59 @@ public class PartitionPlanAllocator { private PartitionPlanAllocator() { } + /** + * Onboard a plugin: promote from buffer and register service allowlists tagged with {@code pluginId}. + */ + public static PartitionPlan onboardPlugin(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap, String updatedBy) { + requireMutationInputs(current, pluginId, partitionCount, updatedBy); + if (current.getPlugins().containsKey(pluginId)) { + assertOnboardNotConflicting(current, pluginId, partitionCount, servicesMap); + throw new PartitionPlanException("Plugin '" + pluginId + "' already has dedicated partitions"); + } + + List remainingBuffer = new ArrayList<>(current.getBuffer().getPartitions()); + List newPluginIds = takeFromBuffer(remainingBuffer, partitionCount); + int topicPartitionCount = appendTailPartitions(newPluginIds, current.getTopicPartitionCount(), partitionCount - newPluginIds.size()); + + Map plugins = addPluginAssignment(current, pluginId, newPluginIds); + Map services = mergeServicesWithPluginId(current.getServices(), servicesMap, pluginId); + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, remainingBuffer, services); + } + + /** + * Update an onboarded plugin: scale tail partitions and/or mutate service allowlists in one version bump. + */ + public static PartitionPlan updatePlugin(PartitionPlan current, String pluginId, UpdatePlugin updateRequest, String updatedBy) { + if (current == null || updateRequest == null) { + throw new PartitionPlanException("Current plan and update request are required"); + } + if (StringUtils.isBlank(pluginId) || StringUtils.isBlank(updatedBy)) { + throw new PartitionPlanException("pluginId and updatedBy are required"); + } + PartitionPlanValidator.validate(current); + + Map services = new LinkedHashMap<>(current.getServices()); + Map plugins = new LinkedHashMap<>(current.getPlugins()); + List bufferIds = new ArrayList<>(current.getBuffer().getPartitions()); + int topicPartitionCount = current.getTopicPartitionCount(); + + applyServiceRemovals(current, services, pluginId, updateRequest.getRemoveServices()); + applyServiceUpdates(current, services, pluginId, updateRequest.getUpdateServices()); + applyServiceAdditions(current, services, pluginId, updateRequest.getAddServices()); + + Integer additionalPartitions = updateRequest.getAdditionalPartitions(); + if (additionalPartitions != null && additionalPartitions >= 1) { + if (!plugins.containsKey(pluginId)) { + throw new PartitionPlanException("Plugin '" + pluginId + "' is not configured; onboard it first"); + } + List pluginIds = new ArrayList<>(plugins.get(pluginId).getPartitions()); + topicPartitionCount = appendTailPartitions(pluginIds, topicPartitionCount, additionalPartitions); + plugins.put(pluginId, new PluginPartitionAssignment(pluginIds)); + } + + return commitPlanUpdate(current, updatedBy, topicPartitionCount, plugins, bufferIds, services); + } + public static PartitionPlan promotePlugin(PartitionPlan current, String pluginId, int partitionCount, String updatedBy) { return promotePlugin(current, pluginId, partitionCount, updatedBy, null, null); } @@ -90,6 +145,60 @@ public static PartitionPlan scalePlugin(PartitionPlan current, String pluginId, return commitPlanUpdate(current, updatedBy, topicPartitionCount, addPluginAssignment(current, pluginId, pluginIds), current.getBuffer().getPartitions(), current.getServices()); } + /** + * True when plugin onboard with {@code partitionCount} and optional services already matches the current plan. + */ + public static boolean isOnboardAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap) { + if (current == null || !pluginHasPartitionCount(current, pluginId, partitionCount)) { + return false; + } + if (servicesMap == null || servicesMap.isEmpty()) { + return true; + } + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry existing = current.getServices().get(repo); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + return false; + } + } + return true; + } + + /** + * True when a service-only update request is already satisfied (scale mutations are never treated as no-op). + */ + public static boolean isUpdateAlreadyApplied(PartitionPlan current, String pluginId, UpdatePlugin updateRequest) { + if (current == null || updateRequest == null) { + return false; + } + Integer additionalPartitions = updateRequest.getAdditionalPartitions(); + if (additionalPartitions != null && additionalPartitions >= 1) { + return false; + } + for (String repo : updateRequest.getRemoveServices()) { + if (current.getServices().containsKey(repo.trim())) { + return false; + } + } + for (Map.Entry entry : updateRequest.getAddServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + return false; + } + } + for (Map.Entry entry : updateRequest.getUpdateServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + return false; + } + } + return updateRequest.hasMutationDelta(); + } + /** * True when the plugin is already promoted with {@code partitionCount} slots and, when {@code repo} is set, * the service allowlist already matches {@code allowedUsers}. @@ -105,11 +214,6 @@ public static boolean isPromoteAlreadyApplied(PartitionPlan current, String plug return true; } - /** True when onboard request matches current allowlist and plugin assignment. */ - public static boolean isOnboardAlreadyApplied(PartitionPlan current, String repo, String pluginId, int partitionCount, List allowedUsers) { - return isPromoteAlreadyApplied(current, pluginId, partitionCount, repo, allowedUsers); - } - /** Applies a merged plan with append-only checks against the current plan. */ public static PartitionPlan replacePlan(PartitionPlan current, PartitionPlan proposed) { if (current == null || proposed == null) { @@ -124,6 +228,58 @@ public static PartitionPlan replacePlan(PartitionPlan current, PartitionPlan pro return next; } + private static void applyServiceRemovals(PartitionPlan current, Map services, String pluginId, List removeServices) { + for (String repoName : removeServices) { + String repo = repoName.trim(); + verifyServiceOwnedByPlugin(current, repo, pluginId); + services.remove(repo); + } + } + + private static void applyServiceUpdates(PartitionPlan current, Map services, String pluginId, Map updateServices) { + for (Map.Entry entry : updateServices.entrySet()) { + String repo = entry.getKey().trim(); + verifyServiceOwnedByPlugin(current, repo, pluginId); + services.put(repo, withPluginId(entry.getValue(), pluginId)); + } + } + + private static void applyServiceAdditions(PartitionPlan current, Map services, String pluginId, Map addServices) { + for (Map.Entry entry : addServices.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry existing = current.getServices().get(repo); + if (existing != null) { + assertServiceOwnedByPlugin(existing, repo, pluginId); + } + services.put(repo, withPluginId(entry.getValue(), pluginId)); + } + } + + private static void verifyServiceOwnedByPlugin(PartitionPlan current, String repo, String pluginId) { + ServiceAllowlistEntry existing = current.getServices().get(repo); + if (existing == null) { + throw new PartitionPlanException("Service '" + repo + "' is not configured"); + } + assertServiceOwnedByPlugin(existing, repo, pluginId); + } + + private static void assertServiceOwnedByPlugin(ServiceAllowlistEntry existing, String repo, String pluginId) { + if (existing.getPluginId() != null && !Objects.equals(existing.getPluginId(), pluginId)) { + throw new PartitionPlanException("Service '" + repo + "' belongs to plugin '" + existing.getPluginId() + "'"); + } + } + + private static boolean serviceEntryMatches(ServiceAllowlistEntry existing, ServiceAllowlistEntry expected, String pluginId) { + return existing.hasSameAllowedUsers(expected.getAllowedUsers()) && Objects.equals(existing.getPluginId(), pluginId); + } + + private static ServiceAllowlistEntry withPluginId(ServiceAllowlistEntry entry, String pluginId) { + if (entry == null) { + throw new PartitionPlanException("Service allowlist entry is required"); + } + return new ServiceAllowlistEntry(entry.getAllowedUsers(), entry.getSource(), entry.getNotes(), pluginId); + } + /** Pull up to count partition IDs from the front of the buffer list. */ private static List takeFromBuffer(List bufferIds, int count) { List taken = new ArrayList<>(Math.min(count, bufferIds.size())); @@ -162,6 +318,26 @@ private static PartitionPlan commitPlanUpdate(PartitionPlan current, String upda return next; } + private static Map mergeServicesWithPluginId(Map currentServices, Map servicesMap, String pluginId) { + Map services = new LinkedHashMap<>(currentServices); + if (servicesMap == null || servicesMap.isEmpty()) { + return services; + } + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry tagged = withPluginId(entry.getValue(), pluginId); + ServiceAllowlistEntry existing = services.get(repo); + if (existing != null && !existing.hasSameAllowedUsers(tagged.getAllowedUsers())) { + throw new PartitionPlanException("Service '" + repo + "' already exists with different allowedUsers"); + } + if (existing != null && existing.getPluginId() != null && !Objects.equals(existing.getPluginId(), pluginId)) { + throw new PartitionPlanException("Service '" + repo + "' belongs to plugin '" + existing.getPluginId() + "'"); + } + services.put(repo, tagged); + } + return services; + } + private static Map mergeServiceAllowlist(Map currentServices, String repo, List allowedUsers) { Map services = new LinkedHashMap<>(currentServices); if (StringUtils.isNotBlank(repo) && allowedUsers != null && !allowedUsers.isEmpty()) { @@ -175,6 +351,22 @@ private static boolean pluginHasPartitionCount(PartitionPlan current, String plu return assignment != null && assignment.getPartitions().size() == partitionCount; } + private static void assertOnboardNotConflicting(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap) { + PluginPartitionAssignment existing = requireNonNull(current.getPlugins().get(pluginId)); + if (existing.getPartitions().size() != partitionCount) { + throw new PartitionPlanException("Plugin '" + pluginId + "' already has " + existing.getPartitions().size() + " dedicated partition(s); requested " + partitionCount); + } + if (servicesMap != null) { + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry serviceEntry = current.getServices().get(repo); + if (serviceEntry != null && !serviceEntry.hasSameAllowedUsers(entry.getValue().getAllowedUsers())) { + throw new PartitionPlanException("Service '" + repo + "' already exists with different allowedUsers"); + } + } + } + } + private static void assertPromoteNotConflicting(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { PluginPartitionAssignment existing = requireNonNull(current.getPlugins().get(pluginId)); if (existing.getPartitions().size() != partitionCount) { diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java index 251c89282b0..d32665b3195 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java @@ -22,61 +22,77 @@ import org.apache.commons.lang3.StringUtils; import org.apache.ranger.audit.producer.kafka.partition.constants.PartitionPlanConstants; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import java.util.List; +import java.util.Map; /** Validates partition-plan REST mutation request bodies before registry writes. */ public class PartitionPlanRequestValidator { private PartitionPlanRequestValidator() { } - public static void validatePatchRequest(PartitionPlanReplacement partitionPlanUpdate) { - if (partitionPlanUpdate == null) { - throw new PartitionPlanException("Partition plan patch request is required"); + public static void validateOnboardPlugin(OnboardPlugin onboardPluginRequest) { + if (onboardPluginRequest == null) { + throw new PartitionPlanException("Onboard plugin request is required"); } - validateExpectedVersion(partitionPlanUpdate.getExpectedVersion()); - if (!partitionPlanUpdate.hasMergeDelta()) { - throw new PartitionPlanException( - "At least one of topicPartitionCount, plugins, buffer, or services must be provided"); + validateNonBlankPluginId(onboardPluginRequest.getPluginId()); + validatePositiveCount(onboardPluginRequest.getPartitionCount(), "partitionCount"); + validateExpectedVersion(onboardPluginRequest.getExpectedVersion()); + validateRequiredServiceEntries(onboardPluginRequest.getServices()); + } + + public static void validateUpdatePlugin(String pluginId, UpdatePlugin updatePluginRequest) { + if (updatePluginRequest == null) { + throw new PartitionPlanException("Update plugin request is required"); + } + validateNonBlankPluginId(pluginId); + validateExpectedVersion(updatePluginRequest.getExpectedVersion()); + if (!updatePluginRequest.hasMutationDelta()) { + throw new PartitionPlanException("At least one of additionalPartitions, addServices, updateServices, or removeServices must be provided"); + } + Integer additionalPartitions = updatePluginRequest.getAdditionalPartitions(); + if (additionalPartitions != null && additionalPartitions < 1) { + throw new PartitionPlanException("additionalPartitions must be >= 1"); } + validateOptionalServiceEntries(updatePluginRequest.getAddServices()); + validateOptionalServiceEntries(updatePluginRequest.getUpdateServices()); + validateRemoveServiceNames(updatePluginRequest.getRemoveServices()); } - public static void validatePromotePlugin(PromotePlugin promotePluginRequest) { - if (promotePluginRequest == null) { - throw new PartitionPlanException("Promote plugin request is required"); - } - validateNonBlankPluginId(promotePluginRequest.getPluginId()); - validatePositiveCount(promotePluginRequest.getPartitionCount(), "partitionCount"); - validateExpectedVersion(promotePluginRequest.getExpectedVersion()); - if (StringUtils.isNotBlank(promotePluginRequest.getRepo()) - && (promotePluginRequest.getAllowedUsers() == null - || promotePluginRequest.getAllowedUsers().isEmpty())) { - throw new PartitionPlanException("allowedUsers are required when repo is specified"); + private static void validateRequiredServiceEntries(Map services) { + if (services == null || services.isEmpty()) { + throw new PartitionPlanException("services are required"); } + validateServiceEntries(services); } - public static void validateScalePlugin(String pluginId, PluginScale scalePlugin) { - if (scalePlugin == null) { - throw new PartitionPlanException("Plugin scale request is required"); + private static void validateOptionalServiceEntries(Map services) { + if (services == null || services.isEmpty()) { + return; } - validateNonBlankPluginId(pluginId); - validatePositiveCount(scalePlugin.getAdditionalPartitions(), "additionalPartitions"); - validateExpectedVersion(scalePlugin.getExpectedVersion()); + validateServiceEntries(services); } - public static void validateOnboardService(OnboardService onboardServiceRequest) { - if (onboardServiceRequest == null) { - throw new PartitionPlanException("Onboard service request is required"); - } - validateNonBlankServiceName(onboardServiceRequest.getServiceName()); - validateNonBlankPluginId(onboardServiceRequest.getPluginId()); - validatePositiveCount(onboardServiceRequest.getPartitionCount(), "partitionCount"); - validateNonEmptyAllowedUsers(onboardServiceRequest.getAllowedUsers()); - validateExpectedVersion(onboardServiceRequest.getExpectedVersion()); + private static void validateServiceEntries(Map services) { + for (Map.Entry entry : services.entrySet()) { + validateNonBlankServiceName(entry.getKey()); + if (entry.getValue() == null) { + throw new PartitionPlanException("Service allowlist entry is required for '" + entry.getKey() + "'"); + } + validateNonEmptyAllowedUsers(entry.getValue().getAllowedUsers()); + } + } + + private static void validateRemoveServiceNames(List removeServices) { + if (removeServices == null || removeServices.isEmpty()) { + return; + } + for (String serviceName : removeServices) { + validateNonBlankServiceName(serviceName); + } } private static void validateExpectedVersion(int expectedVersion) { diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java index 591afe8ed03..200d0bf9de3 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -21,11 +21,9 @@ import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; @@ -69,64 +67,43 @@ public PartitionPlan getPartitionPlan() { return plan; } - /** Merges a partial plan delta via REST PATCH with optimistic locking. */ - public PartitionPlan mergePartitionPlan(PartitionPlanReplacement partitionPlanUpdate, String updatedBy) { - PartitionPlanRequestValidator.validatePatchRequest(partitionPlanUpdate); + /** Onboards a plugin: promote from buffer and register service allowlists atomically. */ + public PartitionPlan onboardPlugin(OnboardPlugin onboardPluginRequest, String updatedBy) { + PartitionPlanRequestValidator.validateOnboardPlugin(onboardPluginRequest); requireDynamicEnabled(); String auditTopic = resolveAuditTopicName(); try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { PartitionPlan currentPlan = requirePlan(registry, auditTopic); - requireExpectedVersion(currentPlan, partitionPlanUpdate.getExpectedVersion()); - PartitionPlan mergedPlan = partitionPlanUpdate.toMergedPlan(currentPlan, updatedBy); - if (currentPlan.sameContentAs(mergedPlan)) { + requireExpectedVersion(currentPlan, onboardPluginRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isOnboardAlreadyApplied(currentPlan, onboardPluginRequest.getPluginId(), onboardPluginRequest.getPartitionCount(), onboardPluginRequest.getServices())) { return returnCurrentPlanNoOp(currentPlan); } - PartitionPlan nextPlan = PartitionPlanAllocator.replacePlan(currentPlan, mergedPlan); - return publishMutation(registry, auditTopic, partitionPlanUpdate.getExpectedVersion(), currentPlan, nextPlan); + PartitionPlan nextPlan = PartitionPlanAllocator.onboardPlugin(currentPlan, onboardPluginRequest.getPluginId(), onboardPluginRequest.getPartitionCount(), onboardPluginRequest.getServices(), updatedBy); + return publishMutation(registry, auditTopic, onboardPluginRequest.getExpectedVersion(), currentPlan, nextPlan); } catch (PartitionPlanException e) { throw e; } catch (Exception e) { - throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); + throw new PartitionPlanException("Failed to onboard plugin in partition plan for audit topic '" + auditTopic + "'", e); } } - /** Promotes a plugin from the buffer to dedicated partitions. */ - public PartitionPlan promotePlugin(PromotePlugin promotePluginRequest, String updatedBy) { - PartitionPlanRequestValidator.validatePromotePlugin(promotePluginRequest); + /** Updates an onboarded plugin: scale and/or mutate service allowlists in one plan version. */ + public PartitionPlan updatePlugin(String pluginId, UpdatePlugin updatePluginRequest, String updatedBy) { + PartitionPlanRequestValidator.validateUpdatePlugin(pluginId, updatePluginRequest); requireDynamicEnabled(); String auditTopic = resolveAuditTopicName(); try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { PartitionPlan currentPlan = requirePlan(registry, auditTopic); - requireExpectedVersion(currentPlan, promotePluginRequest.getExpectedVersion()); - if (PartitionPlanAllocator.isPromoteAlreadyApplied(currentPlan, promotePluginRequest.getPluginId(), promotePluginRequest.getPartitionCount(), promotePluginRequest.getRepo(), promotePluginRequest.getAllowedUsers())) { + requireExpectedVersion(currentPlan, updatePluginRequest.getExpectedVersion()); + if (PartitionPlanAllocator.isUpdateAlreadyApplied(currentPlan, pluginId, updatePluginRequest)) { return returnCurrentPlanNoOp(currentPlan); } - PartitionPlan nextPlan = PartitionPlanAllocator.promotePlugin(currentPlan, promotePluginRequest.getPluginId(), promotePluginRequest.getPartitionCount(), updatedBy, promotePluginRequest.getRepo(), promotePluginRequest.getAllowedUsers()); - return publishMutation(registry, auditTopic, promotePluginRequest.getExpectedVersion(), currentPlan, nextPlan); + PartitionPlan nextPlan = PartitionPlanAllocator.updatePlugin(currentPlan, pluginId, updatePluginRequest, updatedBy); + return publishMutation(registry, auditTopic, updatePluginRequest.getExpectedVersion(), currentPlan, nextPlan); } catch (PartitionPlanException e) { throw e; } catch (Exception e) { - throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); - } - } - - /** Onboards a service repo: upsert allowlist and promote plugin partitions in one plan version. */ - public PartitionPlan onboardService(OnboardService onboardServiceRequest, String updatedBy) { - PartitionPlanRequestValidator.validateOnboardService(onboardServiceRequest); - requireDynamicEnabled(); - String auditTopic = resolveAuditTopicName(); - try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { - PartitionPlan currentPlan = requirePlan(registry, auditTopic); - requireExpectedVersion(currentPlan, onboardServiceRequest.getExpectedVersion()); - if (PartitionPlanAllocator.isOnboardAlreadyApplied(currentPlan, onboardServiceRequest.getServiceName(), onboardServiceRequest.getPluginId(), onboardServiceRequest.getPartitionCount(), onboardServiceRequest.getAllowedUsers())) { - return returnCurrentPlanNoOp(currentPlan); - } - PartitionPlan nextPlan = PartitionPlanAllocator.onboardRepo(currentPlan, onboardServiceRequest.getServiceName(), onboardServiceRequest.getPluginId(), onboardServiceRequest.getPartitionCount(), onboardServiceRequest.getAllowedUsers(), updatedBy); - return publishMutation(registry, auditTopic, onboardServiceRequest.getExpectedVersion(), currentPlan, nextPlan); - } catch (PartitionPlanException e) { - throw e; - } catch (Exception e) { - throw new PartitionPlanException("Failed to onboard repo in partition plan for audit topic '" + auditTopic + "'", e); + throw new PartitionPlanException("Failed to update plugin in partition plan for audit topic '" + auditTopic + "'", e); } } @@ -135,22 +112,6 @@ public Set getPartitionPlanAdminUsers() { return PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); } - /** Appends tail partitions to a plugin already present in the plan. */ - public PartitionPlan scalePlugin(String pluginId, PluginScale scalePlugin, String updatedBy) { - PartitionPlanRequestValidator.validateScalePlugin(pluginId, scalePlugin); - requireDynamicEnabled(); - String auditTopic = resolveAuditTopicName(); - try (PartitionPlanRegistry registry = registryFactory.open(configProps, INGESTOR_PROP_PREFIX)) { - PartitionPlan currentPlan = requirePlan(registry, auditTopic); - PartitionPlan nextPlan = PartitionPlanAllocator.scalePlugin(currentPlan, pluginId, scalePlugin.getAdditionalPartitions(), updatedBy); - return publishMutation(registry, auditTopic, scalePlugin.getExpectedVersion(), currentPlan, nextPlan); - } catch (PartitionPlanException e) { - throw e; - } catch (Exception e) { - throw new PartitionPlanException("Failed to update partition plan for audit topic '" + auditTopic + "'", e); - } - } - /** Validates version, grows the audit topic if needed, writes the plan, and reloads memory. */ private PartitionPlan publishMutation(PartitionPlanRegistry registry, String auditTopic, int expectedVersion, PartitionPlan current, PartitionPlan next) { requireExpectedVersion(current, expectedVersion); diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java index 5c57b93947d..c7d8aca0df2 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java @@ -64,7 +64,7 @@ public static Map loadAllowlistsFromProperties(Pr } allowlistEntriesByRepo.put( serviceRepoName.trim(), - new ServiceAllowlistEntry(allowedUserShortNames, ALLOWLIST_SOURCE_SITE_XML, null)); + new ServiceAllowlistEntry(allowedUserShortNames, ALLOWLIST_SOURCE_SITE_XML, null, null)); } return allowlistEntriesByRepo; } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java new file mode 100644 index 00000000000..1f21b7b57ae --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardPlugin.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Request body for POST /api/audit/partition-plan/plugins. {@code services} is required (non-empty) at validation. */ +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class OnboardPlugin implements Serializable { + private final String pluginId; + private final int partitionCount; + private final int expectedVersion; + private final Map services; + + @JsonCreator + public OnboardPlugin(@JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("services") Map services) { + this.pluginId = pluginId; + this.partitionCount = partitionCount; + this.expectedVersion = expectedVersion; + this.services = copyServices(services); + } + + public OnboardPlugin(String pluginId, int partitionCount, int expectedVersion) { + this(pluginId, partitionCount, expectedVersion, null); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public String getPluginId() { + return pluginId; + } + + public int getPartitionCount() { + return partitionCount; + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public Map getServices() { + return services; + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java index b461afde267..8ad7f9663d5 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/ServiceAllowlistEntry.java @@ -38,20 +38,26 @@ public class ServiceAllowlistEntry implements Serializable { private final List allowedUsers; private final String source; private final String notes; + private final String pluginId; @JsonCreator - public ServiceAllowlistEntry(@JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("source") String source, @JsonProperty("notes") String notes) { + public ServiceAllowlistEntry(@JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("source") String source, @JsonProperty("notes") String notes, @JsonProperty("pluginId") String pluginId) { this.allowedUsers = copyAllowedUsers(allowedUsers); this.source = source; this.notes = notes; + this.pluginId = pluginId; } public static ServiceAllowlistEntry ofUsers(String... users) { - return new ServiceAllowlistEntry(List.of(users), null, null); + return new ServiceAllowlistEntry(List.of(users), null, null, null); } public static ServiceAllowlistEntry ofUsers(List users) { - return new ServiceAllowlistEntry(users, null, null); + return new ServiceAllowlistEntry(users, null, null, null); + } + + public static ServiceAllowlistEntry ofUsers(List users, String pluginId) { + return new ServiceAllowlistEntry(users, null, null, pluginId); } private static List copyAllowedUsers(List allowedUsers) { @@ -79,7 +85,11 @@ public String getNotes() { return notes; } - /** True when normalized allowedUsers match (ignores source and notes). */ + public String getPluginId() { + return pluginId; + } + + /** True when normalized allowedUsers match (ignores source, notes, and pluginId). */ public boolean hasSameAllowedUsers(List users) { return Objects.equals(allowedUsers, ofUsers(users).getAllowedUsers()); } @@ -95,11 +105,12 @@ public boolean equals(Object otherServiceAllowlistEntryObj) { ServiceAllowlistEntry otherAllowlistEntry = (ServiceAllowlistEntry) otherServiceAllowlistEntryObj; return Objects.equals(allowedUsers, otherAllowlistEntry.allowedUsers) && Objects.equals(source, otherAllowlistEntry.source) - && Objects.equals(notes, otherAllowlistEntry.notes); + && Objects.equals(notes, otherAllowlistEntry.notes) + && Objects.equals(pluginId, otherAllowlistEntry.pluginId); } @Override public int hashCode() { - return Objects.hash(allowedUsers, source, notes); + return Objects.hash(allowedUsers, source, notes, pluginId); } } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java new file mode 100644 index 00000000000..535971c4e76 --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class UpdatePlugin implements Serializable { + private final int expectedVersion; + private final Integer additionalPartitions; + private final Map addServices; + private final Map updateServices; + private final List removeServices; + + @JsonCreator + public UpdatePlugin(@JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("additionalPartitions") Integer additionalPartitions, @JsonProperty("addServices") Map addServices, @JsonProperty("updateServices") Map updateServices, @JsonProperty("removeServices") List removeServices) { + this.expectedVersion = expectedVersion; + this.additionalPartitions = additionalPartitions; + this.addServices = copyServices(addServices); + this.updateServices = copyServices(updateServices); + this.removeServices = removeServices == null ? Collections.emptyList() : List.copyOf(removeServices); + } + + private static Map copyServices(Map services) { + if (services == null || services.isEmpty()) { + return Collections.emptyMap(); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(services)); + } + + public int getExpectedVersion() { + return expectedVersion; + } + + public Integer getAdditionalPartitions() { + return additionalPartitions; + } + + public Map getAddServices() { + return addServices; + } + + public Map getUpdateServices() { + return updateServices; + } + + public List getRemoveServices() { + return removeServices; + } + + /** True when at least one mutation field is present. */ + public boolean hasMutationDelta() { + if (additionalPartitions != null && additionalPartitions >= 1) { + return true; + } + if (!addServices.isEmpty()) { + return true; + } + if (!updateServices.isEmpty()) { + return true; + } + return !removeServices.isEmpty(); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 317be6d17d4..c39fc1360cd 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -30,11 +30,9 @@ import org.apache.ranger.audit.producer.kafka.partition.ServiceAllowlistResolver; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; @@ -283,44 +281,13 @@ public Response getPartitionPlan(@Context HttpServletRequest httpRequest) { return ret; } - /** Applies a partial update to the partition plan (optimistic lock via expectedVersion). */ - @PATCH - @Path("/partition-plan") - @Consumes("application/json") - @Produces("application/json") - public Response patchPartitionPlan(PartitionPlanReplacement partitionPlanUpdate, @Context HttpServletRequest httpRequest) { - LOG.debug("==> AuditREST.patchPartitionPlan()"); - Response ret; - if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { - ret = partitionPlanDisabled("PATCH /partition-plan"); - } else { - Response authFailure = authorizePartitionPlanAdmin(httpRequest, "PATCH /partition-plan"); - if (authFailure != null) { - ret = authFailure; - } else { - try { - ret = toSuccessfulPartitionPlanResponse(partitionPlanService.mergePartitionPlan(partitionPlanUpdate, resolveUpdatedBy(httpRequest))); - } catch (PartitionPlanConflictException e) { - ret = toPartitionPlanConflictResponse("PATCH /partition-plan", e); - } catch (PartitionPlanException e) { - ret = toPartitionPlanErrorResponse("PATCH /partition-plan", e); - } catch (Exception e) { - LOG.error("Unexpected error patching partition plan", e); - ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to patch partition plan")).build(); - } - } - } - LOG.debug("<== AuditREST.patchPartitionPlan(): status={}", ret.getStatus()); - return ret; - } - - /** Promotes a plugin from the buffer to dedicated partitions. */ + /** Onboards a plugin from the buffer and registers service allowlists. */ @POST @Path("/partition-plan/plugins") @Consumes("application/json") @Produces("application/json") - public Response createPluginAssignment(PromotePlugin request, @Context HttpServletRequest httpRequest) { - LOG.debug("==> AuditREST.createPluginAssignment(pluginId={})", request != null ? request.getPluginId() : null); + public Response onboardPlugin(OnboardPlugin request, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.onboardPlugin(pluginId={})", request != null ? request.getPluginId() : null); Response ret; if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { ret = partitionPlanDisabled("POST /partition-plan/plugins"); @@ -330,28 +297,28 @@ public Response createPluginAssignment(PromotePlugin request, @Context HttpServl ret = authFailure; } else { try { - ret = toSuccessfulPartitionPlanResponse(partitionPlanService.promotePlugin(request, resolveUpdatedBy(httpRequest))); + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.onboardPlugin(request, resolveUpdatedBy(httpRequest))); } catch (PartitionPlanConflictException e) { ret = toPartitionPlanConflictResponse("POST /partition-plan/plugins", e); } catch (PartitionPlanException e) { ret = toPartitionPlanErrorResponse("POST /partition-plan/plugins", e); } catch (Exception e) { - LOG.error("Unexpected error creating plugin assignment in partition plan", e); - ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to create plugin assignment in partition plan")).build(); + LOG.error("Unexpected error onboarding plugin in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to onboard plugin in partition plan")).build(); } } } - LOG.debug("<== AuditREST.createPluginAssignment(): status={}", ret.getStatus()); + LOG.debug("<== AuditREST.onboardPlugin(): status={}", ret.getStatus()); return ret; } - /** Appends tail partitions to an existing plugin assignment. */ + /** Updates an onboarded plugin: scale partitions and/or mutate service allowlists. */ @PATCH @Path("/partition-plan/plugins/{pluginId}") @Consumes("application/json") @Produces("application/json") - public Response scalePluginAssignment(@PathParam("pluginId") String pluginId, PluginScale pluginScale, @Context HttpServletRequest httpRequest) { - LOG.debug("==> AuditREST.scalePluginAssignment(pluginId={})", pluginId); + public Response updatePlugin(@PathParam("pluginId") String pluginId, UpdatePlugin updateRequest, @Context HttpServletRequest httpRequest) { + LOG.debug("==> AuditREST.updatePlugin(pluginId={})", pluginId); Response ret; if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { ret = partitionPlanDisabled("PATCH /partition-plan/plugins/{pluginId}"); @@ -361,50 +328,18 @@ public Response scalePluginAssignment(@PathParam("pluginId") String pluginId, Pl ret = authFailure; } else { try { - ret = toSuccessfulPartitionPlanResponse(partitionPlanService.scalePlugin(pluginId, pluginScale, resolveUpdatedBy(httpRequest))); + ret = toSuccessfulPartitionPlanResponse(partitionPlanService.updatePlugin(pluginId, updateRequest, resolveUpdatedBy(httpRequest))); } catch (PartitionPlanConflictException e) { ret = toPartitionPlanConflictResponse("PATCH /partition-plan/plugins/" + pluginId, e); } catch (PartitionPlanException e) { ret = toPartitionPlanErrorResponse("PATCH /partition-plan/plugins/" + pluginId, e); } catch (Exception e) { - LOG.error("Unexpected error scaling plugin assignment in partition plan", e); - ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to scale plugin assignment in partition plan")).build(); - } - } - } - LOG.debug("<== AuditREST.scalePluginAssignment(): status={}", ret.getStatus()); - return ret; - } - - /** Onboards a service: upsert allowlist and promote plugin partitions in one plan version. */ - @POST - @Path("/partition-plan/services") - @Consumes("application/json") - @Produces("application/json") - public Response createService(OnboardService request, @Context HttpServletRequest httpRequest) { - LOG.debug("==> AuditREST.createService(serviceName={}, pluginId={})", - request != null ? request.getServiceName() : null, request != null ? request.getPluginId() : null); - Response ret; - if (!partitionPlanService.isDynamicPartitionPlanEnabled()) { - ret = partitionPlanDisabled("POST /partition-plan/services"); - } else { - Response authFailure = authorizePartitionPlanAdmin(httpRequest, "POST /partition-plan/services"); - if (authFailure != null) { - ret = authFailure; - } else { - try { - ret = toSuccessfulPartitionPlanResponse(partitionPlanService.onboardService(request, resolveUpdatedBy(httpRequest))); - } catch (PartitionPlanConflictException e) { - ret = toPartitionPlanConflictResponse("POST /partition-plan/services", e); - } catch (PartitionPlanException e) { - ret = toPartitionPlanErrorResponse("POST /partition-plan/services", e); - } catch (Exception e) { - LOG.error("Unexpected error creating service in partition plan", e); - ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to create service in partition plan")).build(); + LOG.error("Unexpected error updating plugin in partition plan", e); + ret = Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(buildErrorResponse("Failed to update plugin in partition plan")).build(); } } } - LOG.debug("<== AuditREST.createService(): status={}", ret.getStatus()); + LOG.debug("<== AuditREST.updatePlugin(): status={}", ret.getStatus()); return ret; } diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java index 0c416ac6084..220366ac407 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocatorTest.java @@ -19,10 +19,14 @@ import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -50,6 +54,50 @@ public void testPromotePluginFromBuffer() { assertIterableEquals(List.of(3, 4, 5), next.getPlugins().get("hiveServer2").getPartitions()); } + @Test + public void testOnboardPluginWithMultipleServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + + PartitionPlan next = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); + + assertEquals(2, next.getVersion()); + assertIterableEquals(List.of(6, 7, 8), next.getPlugins().get("trino").getPartitions()); + assertEquals("trino", next.getServices().get("dev_hive").getPluginId()); + assertEquals("trino", next.getServices().get("dev_hive2").getPluginId()); + assertIterableEquals(List.of("hive"), next.getServices().get("dev_hive").getAllowedUsers()); + assertIterableEquals(List.of("hive2"), next.getServices().get("dev_hive2").getAllowedUsers()); + } + + @Test + public void testUpdatePluginAddsAndRemovesServices() { + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + PartitionPlan onboarded = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); + + Map addServices = Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")); + UpdatePlugin update = new UpdatePlugin(onboarded.getVersion(), null, addServices, null, List.of("dev_hive2")); + PartitionPlan updated = PartitionPlanAllocator.updatePlugin(onboarded, "trino", update, "ops"); + + assertEquals(3, updated.getVersion()); + assertTrue(updated.getServices().containsKey("dev_hive")); + assertTrue(updated.getServices().containsKey("dev_hive3")); + assertFalse(updated.getServices().containsKey("dev_hive2")); + assertEquals("trino", updated.getServices().get("dev_hive3").getPluginId()); + } + + @Test + public void testUpdatePluginScalesViaAdditionalPartitions() { + UpdatePlugin update = new UpdatePlugin(initialPlan.getVersion(), 3, null, null, null); + PartitionPlan scaled = PartitionPlanAllocator.updatePlugin(initialPlan, "hiveServer2", update, "ops"); + + assertEquals(2, scaled.getVersion()); + assertEquals(18, scaled.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16, 17), scaled.getPlugins().get("hiveServer2").getPartitions()); + } + @Test public void testPromotePluginGrowsTopicWhenBufferInsufficient() { PartitionPlan next = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 12, "ops"); @@ -79,19 +127,12 @@ public void testPromoteAlreadyConfiguredPluginFails() { } @Test - public void testIsPromoteAlreadyAppliedWhenPluginAndCountMatch() { - PartitionPlan promoted = PartitionPlanAllocator.promotePlugin(initialPlan, "trino", 3, "ops"); - - assertTrue(PartitionPlanAllocator.isPromoteAlreadyApplied(promoted, "trino", 3, null, null)); - assertFalse(PartitionPlanAllocator.isPromoteAlreadyApplied(promoted, "trino", 5, null, null)); - } - - @Test - public void testIsOnboardAlreadyAppliedWhenAllowlistAndPluginMatch() { - PartitionPlan onboarded = PartitionPlanAllocator.onboardRepo(initialPlan, "dev_trino", "trino", 3, List.of("trino"), "ops"); + public void testIsOnboardAlreadyAppliedWhenPluginServicesAndCountMatch() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + PartitionPlan onboarded = PartitionPlanAllocator.onboardPlugin(initialPlan, "trino", 3, services, "ops"); - assertTrue(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "dev_trino", "trino", 3, List.of("trino"))); - assertFalse(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "dev_trino", "trino", 3, List.of("other"))); + assertTrue(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "trino", 3, services)); + assertFalse(PartitionPlanAllocator.isOnboardAlreadyApplied(onboarded, "trino", 5, services)); } @Test @@ -104,6 +145,19 @@ public void testPromoteConflictWhenPartitionCountDiffers() { assertTrue(error.getMessage().contains("requested 5")); } + @Test + public void testUpdatePluginRejectsRemoveForForeignService() { + PartitionPlan withTaggedService = initialPlan.toBuilder() + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers(List.of("hive"), "hdfs"))) + .build(); + + UpdatePlugin update = new UpdatePlugin(withTaggedService.getVersion(), null, null, null, List.of("dev_hive")); + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanAllocator.updatePlugin(withTaggedService, "hiveServer2", update, "ops")); + + assertTrue(error.getMessage().contains("belongs to plugin")); + } + @Test public void testScaleUnknownPluginFails() { assertThrows(PartitionPlanException.class, () -> PartitionPlanAllocator.scalePlugin(initialPlan, "trino", 2, "ops")); diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java index 9aa4b0de80a..b0040b74636 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolderTest.java @@ -83,7 +83,7 @@ public void testGetAllowedUsersReturnsRegistryUsersForOnboardedRepo() { @Test public void testInstallRejectsEmptyServiceAllowlist() { - Map services = Map.of("dev_hdfs", new ServiceAllowlistEntry(List.of(), "test", null)); + Map services = Map.of("dev_hdfs", new ServiceAllowlistEntry(List.of(), "test", null, null)); PartitionPlan plan = basePlanBuilder().services(services).build(); assertThrows(PartitionPlanException.class, () -> PartitionPlanHolder.getInstance().install(plan, TOPIC_PARTITIONS)); diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java index d8c94698b87..481e73ec500 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java @@ -18,13 +18,12 @@ package org.apache.ranger.audit.producer.kafka.partition; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.junit.jupiter.api.Test; import java.util.List; @@ -134,21 +133,35 @@ public void testSameContentAsIgnoresVersionAndMetadata() { } @Test - public void testRequestValidatorRejectsBlankPromotePluginId() { + public void testRequestValidatorRejectsBlankOnboardPluginId() { assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validatePromotePlugin(new PromotePlugin("", 2, 1))); + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("", 2, 1))); + } + + @Test + public void testRequestValidatorRejectsOnboardMissingServices() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1))); + + assertTrue(error.getMessage().contains("services are required")); } @Test public void testRequestValidatorRejectsOnboardServiceWithoutAllowedUsers() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of(" "))); + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1, services))); + } + + @Test + public void testRequestValidatorRejectsUpdateWithZeroAdditionalPartitions() { assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateOnboardService( - new OnboardService("dev_trino", "trino", 2, List.of(" "), 1))); + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null))); } @Test - public void testRequestValidatorRejectsScaleWithZeroAdditionalPartitions() { + public void testRequestValidatorRejectsUpdateWithoutMutationDelta() { assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateScalePlugin("hiveServer2", new PluginScale(0, 1))); + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, null, null, null, null))); } } diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java new file mode 100644 index 00000000000..466e5463c58 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidatorTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.producer.kafka.partition; + +import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; +import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class PartitionPlanRequestValidatorTest { + @Test + public void testValidateOnboardPluginAcceptsNonEmptyServices() { + Map services = Map.of( + "dev_hive", ServiceAllowlistEntry.ofUsers("hive"), + "dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + + assertDoesNotThrow(() -> PartitionPlanRequestValidator.validateOnboardPlugin( + new OnboardPlugin("hiveServer2", 3, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsNullRequest() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(null)); + } + + @Test + public void testValidateOnboardPluginRejectsMissingServices() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1))); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testValidateOnboardPluginRejectsEmptyServicesMap() { + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin( + new OnboardPlugin("trino", 2, 1, Collections.emptyMap()))); + + assertTrue(error.getMessage().contains("services are required")); + } + + @Test + public void testValidateOnboardPluginRejectsBlankPluginId() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("", 2, 1))); + } + + @Test + public void testValidateOnboardPluginRejectsNonPositivePartitionCount() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 0, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsBlankServiceName() { + Map services = new LinkedHashMap<>(); + services.put(" ", ServiceAllowlistEntry.ofUsers("hive")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("hiveServer2", 2, 1, services))); + } + + @Test + public void testValidateOnboardPluginRejectsServiceWithoutAllowedUsers() { + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of(" "))); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1, services))); + } + + @Test + public void testValidateUpdatePluginAcceptsAddServicesOnly() { + UpdatePlugin update = new UpdatePlugin(1, null, Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")), null, null); + + assertDoesNotThrow(() -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", update)); + } + + @Test + public void testValidateUpdatePluginRejectsZeroAdditionalPartitions() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null))); + } + + @Test + public void testValidateUpdatePluginRejectsWithoutMutationDelta() { + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, null, null, null, null))); + } + + @Test + public void testValidateUpdatePluginRejectsBlankRemoveServiceName() { + UpdatePlugin update = new UpdatePlugin(1, null, null, null, List.of(" ")); + + assertThrows(PartitionPlanException.class, + () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", update)); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java index b2b70290096..5e270e62cf2 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanServiceMutationTest.java @@ -19,18 +19,16 @@ import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanConflictException; import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardService; +import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginScale; -import org.apache.ranger.audit.producer.kafka.partition.model.PromotePlugin; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,6 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -58,136 +57,119 @@ public void tearDown() { } @Test - public void testPromotePluginPublishesNextVersion() throws Exception { + public void testOnboardPluginPublishesNextVersion() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - PartitionPlan result = service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops"); + PartitionPlan result = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, trinoServices()), "ops"); assertEquals(2, result.getVersion()); assertEquals(2, registry.getPlan().getVersion()); assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); + assertEquals("trino", result.getServices().get("dev_trino").getPluginId()); assertEquals(1, registry.getWriteCount()); assertEquals(result, PartitionPlanHolder.getInstance().getPlan()); } @Test - public void testMergePlanAddsNewPluginOnly() throws Exception { - MutableRegistry registry = new MutableRegistry(initialPlan); - PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - PartitionPlanReplacement request = new PartitionPlanReplacement( - 1, - null, - Map.of("trino", PluginPartitionAssignment.of(6, 7, 8)), - null, - null); - - PartitionPlan result = service.mergePartitionPlan(request, "ops"); - - assertEquals(2, result.getVersion()); - assertIterableEquals(List.of(0, 1, 2), result.getPlugins().get("hdfs").getPartitions()); - assertIterableEquals(List.of(3, 4, 5), result.getPlugins().get("hiveServer2").getPartitions()); - assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); - assertIterableEquals(List.of(9, 10, 11, 12, 13, 14), result.getBuffer().getPartitions()); - } - - @Test - public void testScalePluginAppendsTailPartitionsViaDedicatedEndpoint() throws Exception { + public void testOnboardPluginWithMultipleServices() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); - PartitionPlan result = service.scalePlugin("hiveServer2", new PluginScale(2, 1), "ops"); + PartitionPlan result = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops"); assertEquals(2, result.getVersion()); - assertEquals(17, result.getTopicPartitionCount()); - assertIterableEquals(List.of(3, 4, 5, 15, 16), result.getPlugins().get("hiveServer2").getPartitions()); + assertEquals("trino", result.getServices().get("dev_hive").getPluginId()); + assertEquals("trino", result.getServices().get("dev_hive2").getPluginId()); } @Test - public void testMergePlanRejectsExistingPluginDelta() { + public void testUpdatePluginAddsAndRemovesServices() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - PartitionPlanReplacement request = new PartitionPlanReplacement( - 1, - null, - Map.of("hiveServer2", PluginPartitionAssignment.of(3, 4, 5, 15, 16)), - null, - null); + Map services = new LinkedHashMap<>(); + services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); + services.put("dev_hive2", ServiceAllowlistEntry.ofUsers("hive2")); + PartitionPlan onboarded = service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops"); - PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.mergePartitionPlan(request, "ops")); + UpdatePlugin update = new UpdatePlugin(onboarded.getVersion(), null, Map.of("dev_hive3", ServiceAllowlistEntry.ofUsers("hive3")), null, List.of("dev_hive2")); + PartitionPlan updated = service.updatePlugin("trino", update, "ops"); - assertTrue(error.getMessage().contains("different partition assignment")); + assertEquals(3, updated.getVersion()); + assertTrue(updated.getServices().containsKey("dev_hive")); + assertTrue(updated.getServices().containsKey("dev_hive3")); + assertFalse(updated.getServices().containsKey("dev_hive2")); } @Test - public void testMergePlanCanAddServices() throws Exception { + public void testUpdatePluginScalesViaAdditionalPartitions() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - Map services = new LinkedHashMap<>(); - services.put("dev_new_hive", ServiceAllowlistEntry.ofUsers("hive")); - PartitionPlanReplacement request = new PartitionPlanReplacement(1, null, null, null, services); - PartitionPlan result = service.mergePartitionPlan(request, "ops"); + PartitionPlan result = service.updatePlugin("hiveServer2", new UpdatePlugin(1, 2, null, null, null), "ops"); assertEquals(2, result.getVersion()); - assertIterableEquals(List.of("hive"), result.getServices().get("dev_new_hive").getAllowedUsers()); + assertEquals(17, result.getTopicPartitionCount()); + assertIterableEquals(List.of(3, 4, 5, 15, 16), result.getPlugins().get("hiveServer2").getPartitions()); } @Test - public void testPromotePluginRejectsBlankPluginId() { + public void testOnboardPluginRejectsEmptyServicesMap() { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.promotePlugin(new PromotePlugin(" ", 3, 1), "ops")); + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, Collections.emptyMap()), "ops")); - assertTrue(error.getMessage().contains("pluginId")); + assertTrue(error.getMessage().contains("services are required")); } @Test - public void testScalePluginRejectsInvalidAdditionalPartitions() { + public void testOnboardPluginRejectsMissingServices() { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.scalePlugin("hiveServer2", new PluginScale(0, 1), "ops")); + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1), "ops")); - assertTrue(error.getMessage().contains("additionalPartitions")); + assertTrue(error.getMessage().contains("services are required")); } @Test - public void testOnboardServiceRejectsMissingAllowedUsers() { + public void testOnboardPluginRejectsBlankPluginId() { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.onboardService(new OnboardService("dev_trino", "trino", 3, List.of(), 1), "ops")); + () -> service.onboardPlugin(new OnboardPlugin(" ", 3, 1), "ops")); - assertTrue(error.getMessage().contains("allowedUsers")); + assertTrue(error.getMessage().contains("pluginId")); } @Test - public void testOnboardRepoPromotesPluginAndUpsertsAllowlist() throws Exception { + public void testUpdatePluginRejectsInvalidAdditionalPartitions() { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - PartitionPlan result = service.onboardService(new OnboardService("dev_trino", "trino", 3, List.of("trino"), 1), "ops"); + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.updatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null), "ops")); - assertEquals(2, result.getVersion()); - assertIterableEquals(List.of(6, 7, 8), result.getPlugins().get("trino").getPartitions()); - assertIterableEquals(List.of("trino"), result.getServices().get("dev_trino").getAllowedUsers()); + assertTrue(error.getMessage().contains("additionalPartitions")); } @Test - public void testPromoteWithAllowlistUpsertsServices() throws Exception { + public void testOnboardPluginRejectsServiceWithoutAllowedUsers() { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of())); - PartitionPlan result = service.promotePlugin(new PromotePlugin("trino", 3, 1, "dev_trino", List.of("trino")), "ops"); + PartitionPlanException error = assertThrows(PartitionPlanException.class, + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, services), "ops")); - assertEquals(2, result.getVersion()); - assertIterableEquals(List.of("trino"), result.getServices().get("dev_trino").getAllowedUsers()); + assertTrue(error.getMessage().contains("allowedUsers")); } @Test @@ -196,7 +178,7 @@ public void testStaleExpectedVersionReturnsConflict() { PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, - () -> service.promotePlugin(new PromotePlugin("trino", 3, 99), "ops")); + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 99, trinoServices()), "ops")); assertEquals(initialPlan, conflict.getCurrentPlan()); assertEquals(0, registry.getWriteCount()); @@ -219,7 +201,7 @@ public PartitionPlan readPlan(String auditTopicKey) { PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); PartitionPlanConflictException conflict = assertThrows(PartitionPlanConflictException.class, - () -> service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops")); + () -> service.onboardPlugin(new OnboardPlugin("trino", 3, 1, trinoServices()), "ops")); assertEquals(peerPlan, conflict.getCurrentPlan()); assertEquals(0, registry.getWriteCount()); @@ -231,7 +213,7 @@ public void testTopicGrowFailureSurfacesAsPlanException() { PartitionPlanService service = service(registry, new FailingAuditTopicPartitionGrower()); PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.promotePlugin(new PromotePlugin("trino", 12, 1), "ops")); + () -> service.onboardPlugin(new OnboardPlugin("trino", 12, 1, trinoServices()), "ops")); assertTrue(error.getMessage().contains("grow audit topic")); assertEquals(0, registry.getWriteCount()); @@ -241,13 +223,14 @@ public void testTopicGrowFailureSurfacesAsPlanException() { public void testDuplicateOnboardReturnsCurrentPlanWithoutWrite() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - OnboardService request = new OnboardService("dev_trino", "trino", 3, List.of("trino"), 1); + Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + OnboardPlugin request = new OnboardPlugin("trino", 3, 1, services); - PartitionPlan first = service.onboardService(request, "ops"); + PartitionPlan first = service.onboardPlugin(request, "ops"); assertEquals(1, registry.getWriteCount()); - OnboardService retry = new OnboardService("dev_trino", "trino", 3, List.of("trino"), first.getVersion()); - PartitionPlan second = service.onboardService(retry, "ops"); + OnboardPlugin retry = new OnboardPlugin("trino", 3, first.getVersion(), services); + PartitionPlan second = service.onboardPlugin(retry, "ops"); assertEquals(first, second); assertEquals(2, second.getVersion()); @@ -255,79 +238,24 @@ public void testDuplicateOnboardReturnsCurrentPlanWithoutWrite() throws Exceptio } @Test - public void testDuplicatePromoteReturnsCurrentPlanWithoutWrite() throws Exception { + public void testUpdatePluginStillAppendsOnRepeatScale() throws Exception { MutableRegistry registry = new MutableRegistry(initialPlan); PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - PromotePlugin request = new PromotePlugin("trino", 3, 1); - PartitionPlan first = service.promotePlugin(request, "ops"); + service.updatePlugin("hiveServer2", new UpdatePlugin(1, 2, null, null, null), "ops"); assertEquals(1, registry.getWriteCount()); - PromotePlugin retry = new PromotePlugin("trino", 3, first.getVersion()); - PartitionPlan second = service.promotePlugin(retry, "ops"); - - assertEquals(first, second); - assertEquals(2, second.getVersion()); - assertEquals(1, registry.getWriteCount()); - } - - @Test - public void testPromoteConflictWhenPartitionCountDiffers() { - MutableRegistry registry = new MutableRegistry(initialPlan); - PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - service.promotePlugin(new PromotePlugin("trino", 3, 1), "ops"); - - PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> service.promotePlugin(new PromotePlugin("trino", 5, 2), "ops")); - - assertTrue(error.getMessage().contains("requested 5")); - assertEquals(1, registry.getWriteCount()); - } - - @Test - public void testMergePlanNoOpWhenServicesUnchanged() throws Exception { - MutableRegistry registry = new MutableRegistry(initialPlan); - PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - Map services = new LinkedHashMap<>(); - services.put("dev_hive", ServiceAllowlistEntry.ofUsers("hive")); - service.mergePartitionPlan(new PartitionPlanReplacement(1, null, null, null, services), "ops"); - assertEquals(1, registry.getWriteCount()); - - PartitionPlan result = service.mergePartitionPlan(new PartitionPlanReplacement(2, null, null, null, services), "ops"); - - assertEquals(2, result.getVersion()); - assertEquals(1, registry.getWriteCount()); - } - - @Test - public void testMergePlanStillAppliesWhenServicesChange() throws Exception { - MutableRegistry registry = new MutableRegistry(initialPlan); - PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - Map services = new LinkedHashMap<>(); - services.put("dev_new_hive", ServiceAllowlistEntry.ofUsers("hive")); - PartitionPlanReplacement request = new PartitionPlanReplacement(1, null, null, null, services); - - PartitionPlan result = service.mergePartitionPlan(request, "ops"); - - assertEquals(2, result.getVersion()); - assertEquals(1, registry.getWriteCount()); - } - - @Test - public void testScalePluginStillAppendsOnRepeat() throws Exception { - MutableRegistry registry = new MutableRegistry(initialPlan); - PartitionPlanService service = service(registry, new NoOpAuditTopicPartitionGrower()); - - service.scalePlugin("hiveServer2", new PluginScale(2, 1), "ops"); - assertEquals(1, registry.getWriteCount()); - - PartitionPlan second = service.scalePlugin("hiveServer2", new PluginScale(2, 2), "ops"); + PartitionPlan second = service.updatePlugin("hiveServer2", new UpdatePlugin(2, 2, null, null, null), "ops"); assertEquals(3, second.getVersion()); assertEquals(2, registry.getWriteCount()); assertIterableEquals(List.of(3, 4, 5, 15, 16, 17, 18), second.getPlugins().get("hiveServer2").getPartitions()); } + private static Map trinoServices() { + return Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino")); + } + private static PartitionPlanService service(MutableRegistry registry, KafkaAuditTopicPartitionGrower topicGrower) { return new PartitionPlanService(enabledConfig(), PartitionPlanHolder.getInstance(), new FixedPartitionPlanRegistryFactory(registry), topicGrower); } diff --git a/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh new file mode 100755 index 00000000000..f33507286e7 --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# E2E helpers: dynamic allowlist + auth_to_local + POST /api/audit/access via curl/SPNEGO. +# Requires partition-plan-e2e-lib.sh (pp_*) to be sourced first. + +DAEL_INGESTOR_HOST="${CONTAINER}.rangernw" +DAEL_INGESTOR_HTTP_PORT="${INGESTOR_HTTP_PORT:-7081}" +DAEL_ACCESS_PATH="/api/audit/access" +DAEL_ONBOARD_PATH="/api/audit/partition-plan/plugins" +DAEL_ACCESS_CODE="" +DAEL_ACCESS_BODY="" + +# repo|appId|shortName|source_container|keytab|principal +# shortName = auth_to_local output checked against services[].allowedUsers +# appId = Kafka plugin id (agentId on audit events) +readonly -a DAEL_PLUGIN_PROFILES=( + "dev_hdfs|hdfs|hdfs|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" + "dev_yarn|yarn|yarn|ranger-hadoop|/etc/keytabs/yarn.keytab|yarn/ranger-hadoop.rangernw@EXAMPLE.COM" + "dev_hive|hiveServer2|hive|ranger-hive|/etc/keytabs/hive.keytab|hive/ranger-hive.rangernw@EXAMPLE.COM" + "dev_hbase|hbaseMaster|hbase|ranger-hbase|/etc/keytabs/hbase.keytab|hbase/ranger-hbase.rangernw@EXAMPLE.COM" + "dev_kafka|kafka|kafka|ranger-kafka|/etc/keytabs/kafka.keytab|kafka/ranger-kafka.rangernw@EXAMPLE.COM" + "dev_knox|knox|knox|ranger-knox|/etc/keytabs/knox.keytab|knox/ranger-knox.rangernw@EXAMPLE.COM" + "dev_kms|kms|rangerkms|ranger-kms|/etc/keytabs/rangerkms.keytab|rangerkms/ranger-kms.rangernw@EXAMPLE.COM" + "dev_trino|trino|trino|ranger-trino|/etc/keytabs/trino.keytab|trino/ranger-trino.rangernw@EXAMPLE.COM" + "dev_solr|solr|solr|ranger-solr|/etc/keytabs/solr.keytab|solr/ranger-solr.rangernw@EXAMPLE.COM" + "dev_ozone|ozone|om|ozone-om|/etc/keytabs/om.keytab|om/ranger-ozone.rangernw@EXAMPLE.COM" +) + +dael_ingestor_access_url() { + echo "http://${DAEL_INGESTOR_HOST}:${DAEL_INGESTOR_HTTP_PORT}${DAEL_ACCESS_PATH}" +} + +dael_audit_event_json() { + local repo="$1" + local app_id="$2" + local evt_ms + evt_ms="$(python3 -c 'import time; print(int(time.time()*1000))')" + printf '[{"repo":"%s","reqUser":"e2e-audit-user","evtTime":%s,"access":"read","resource":"/e2e/path","result":1,"agent":"%s"}]' \ + "${repo}" "${evt_ms}" "${app_id}" +} + +dael_container_running() { + local name="$1" + docker ps --filter "name=^${name}$" --filter status=running --format '{{.Names}}' | grep -qx "${name}" +} + +dael_access_post_from_container() { + local src_container="$1" + local keytab="$2" + local principal="$3" + local service_name="$4" + local app_id="$5" + local body + local url + local outfile="/tmp/dael-access-body-${src_container}-$$" + local krb_cc="/tmp/dael-access-krb-${src_container}-$$" + local body_file="/tmp/dael-access-req-${src_container}-$$" + + body="$(dael_audit_event_json "${service_name}" "${app_id}")" + url="$(dael_ingestor_access_url)?serviceName=${service_name}&appId=${app_id}" + + if ! dael_container_running "${src_container}"; then + DAEL_ACCESS_CODE="000" + DAEL_ACCESS_BODY="container ${src_container} not running" + return 1 + fi + + if ! docker exec "${src_container}" test -f "${keytab}" 2>/dev/null; then + DAEL_ACCESS_CODE="000" + DAEL_ACCESS_BODY="keytab missing: ${keytab}" + return 1 + fi + + printf '%s' "${body}" | docker exec -i "${src_container}" tee "${body_file}" >/dev/null + DAEL_ACCESS_CODE="$(docker exec "${src_container}" bash -c \ + "export KRB5CCNAME=${krb_cc}; kinit -kt '${keytab}' '${principal}' >/dev/null 2>&1 && \ +curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X POST \ + -H 'Content-Type: application/json' -d @${body_file} '${url}'")" + DAEL_ACCESS_BODY="$(docker exec "${src_container}" cat "${outfile}" 2>/dev/null || true)" + docker exec "${src_container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true +} + +dael_expect_access() { + local label="$1" + local expect_code="$2" + if [[ "${DAEL_ACCESS_CODE}" == "${expect_code}" ]]; then + pp_record_pass "${label} (HTTP ${expect_code})" + return 0 + fi + pp_record_fail "${label} expected HTTP ${expect_code}, got ${DAEL_ACCESS_CODE}: ${DAEL_ACCESS_BODY}" + return 1 +} + +dael_json_authenticated_user() { + printf '%s' "${DAEL_ACCESS_BODY}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('authenticatedUser',''))" 2>/dev/null || echo "" +} + +dael_plan_update_services_body() { + local plan_json="$1" + local plugin_id="$2" + local repo="$3" + local users_csv="$4" + printf '%s' "${plan_json}" | python3 -c \ + "import sys,json; plan=json.load(sys.stdin); plugin_id,repo,users=sys.argv[1:4]; \ +users=[u.strip() for u in users.split(',') if u.strip()]; \ +body={'expectedVersion': plan['version'], 'updateServices': {repo: {'allowedUsers': users}}}; \ +print(json.dumps(body))" "${plugin_id}" "${repo}" "${users_csv}" +} + +dael_patch_allowed_users() { + local repo="$1" + local plugin_id="$2" + local users_csv="$3" + local patch_url + local patch_body + + patch_url="$(pp_ingestor_plan_url "${CONTAINER}")" + pp_ingestor_request "${CONTAINER}" GET "${patch_url}" + if [[ "${HTTP_CODE}" != "200" ]]; then + pp_record_fail "GET plan before PATCH allowedUsers for ${repo}: ${HTTP_CODE}" + return 1 + fi + patch_body="$(dael_plan_update_services_body "${HTTP_BODY}" "${plugin_id}" "${repo}" "${users_csv}")" + pp_ingestor_request "${CONTAINER}" PATCH "${patch_url}/plugins/${plugin_id}" "${patch_body}" + if [[ "${HTTP_CODE}" == "200" ]]; then + return 0 + fi + pp_record_fail "PATCH updateServices for ${repo} (${plugin_id}) failed: HTTP ${HTTP_CODE} ${HTTP_BODY}" + return 1 +} + +dael_onboard_repo() { + local repo="$1" + local plugin_id="$2" + local short_name="$3" + local partition_count="${4:-2}" + local version="$5" + local url body + + url="$(pp_ingestor_plan_url "${CONTAINER}")/plugins" + body="$(pp_build_onboard_plugin_json "${plugin_id}" "${partition_count}" "${version}" "${repo}:${short_name}")" + pp_ingestor_request "${CONTAINER}" POST "${url}" "${body}" + [[ "${HTTP_CODE}" == "200" ]] +} + +dael_wait_auth_to_local_applied() { + local container="$1" + local timeout="${2:-60}" + local deadline=$((SECONDS + timeout)) + while (( SECONDS < deadline )); do + if docker logs "${container}" 2>&1 | tail -200 | grep -q "Applied composed auth_to_local rules"; then + return 0 + fi + sleep 2 + done + return 1 +} + +dael_write_curl_cookbook() { + local out_file="$1" + local access_url onboard_url + access_url="$(dael_ingestor_access_url)" + onboard_url="http://${DAEL_INGESTOR_HOST}:${DAEL_INGESTOR_HTTP_PORT}${DAEL_ONBOARD_PATH}" + + { + echo "# Ranger audit ingestor access E2E — curl cookbook" + echo "# Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "# Run inside a container on rangernw with the plugin keytab (SPNEGO Host must be FQDN)." + echo "" + echo "# Health (no auth):" + echo "curl -s http://localhost:7081/api/audit/health" + echo "" + echo "# Partition plan (HTTP SPNEGO as ingestor HTTP service):" + echo "kinit -kt /etc/keytabs/HTTP.keytab HTTP/ranger-audit-ingestor.rangernw@EXAMPLE.COM" + echo "curl -s --negotiate -u : http://ranger-audit-ingestor.rangernw:7081/api/audit/partition-plan" + echo "" + echo "# Onboard plugin + allowlist (dynamic mode, services required):" + echo "curl -s --negotiate -u : -X POST -H 'Content-Type: application/json' \\" + echo " '${onboard_url}' \\" + echo " -d '{\"pluginId\":\"trino\",\"partitionCount\":3,\"expectedVersion\":1,\"services\":{\"dev_trino\":{\"allowedUsers\":[\"trino\"]}}}'" + echo "" + + local profile repo app_id short_name src_container keytab principal body + for profile in "${DAEL_PLUGIN_PROFILES[@]}"; do + IFS='|' read -r repo app_id short_name src_container keytab principal <<< "${profile}" + body="$(dael_audit_event_json "${repo}" "${app_id}")" + echo "# POST audit — ${repo} (${short_name} via auth_to_local, appId=${app_id})" + echo "# Run on: ${src_container}" + echo "kinit -kt ${keytab} ${principal}" + echo "curl -s -w '\\nHTTP %{http_code}\\n' --negotiate -u : -X POST \\" + echo " -H 'Content-Type: application/json' \\" + echo " '${access_url}?serviceName=${repo}&appId=${app_id}' \\" + echo " -d '${body}'" + echo "" + done + } > "${out_file}" + echo " Wrote curl cookbook: ${out_file}" +} + +dael_test_plugin_access() { + local profile="$1" + local repo app_id short_name src_container keytab principal auth_user + + IFS='|' read -r repo app_id short_name src_container keytab principal <<< "${profile}" + + if ! dael_container_running "${src_container}"; then + echo " SKIP ${repo}: ${src_container} not running" + return 0 + fi + + echo "" + echo " Plugin ${app_id} / repo ${repo} (shortName=${short_name})..." + dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" + if ! dael_expect_access "${repo} POST /access as ${short_name}" "200"; then + return 1 + fi + auth_user="$(dael_json_authenticated_user)" + if [[ "${auth_user}" == "${short_name}" ]]; then + pp_record_pass "${repo} authenticatedUser=${auth_user}" + else + pp_record_fail "${repo} authenticatedUser expected ${short_name}, got ${auth_user}" + return 1 + fi + return 0 +} + +dael_test_dynamic_user_toggle() { + local repo="$1" + local short_name="$2" + local src_container="$3" + local keytab="$4" + local principal="$5" + local app_id="$6" + + echo "" + echo " Dynamic allowlist toggle for ${repo}..." + + if ! dael_patch_allowed_users "${repo}" "${app_id}" "wronguser-not-allowed"; then + return 1 + fi + if dael_wait_auth_to_local_applied "${CONTAINER}" 30; then + pp_record_pass "auth_to_local recomposed after allowlist changed" + else + pp_record_fail "no auth_to_local recompose log after allowlist changed" + fi + + dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" + dael_expect_access "${repo} denied when user not in allowlist" "403" || return 1 + + if ! dael_patch_allowed_users "${repo}" "${app_id}" "${short_name}"; then + return 1 + fi + if dael_wait_auth_to_local_applied "${CONTAINER}" 30; then + pp_record_pass "auth_to_local recomposed after allowlist restored" + else + pp_record_fail "no auth_to_local recompose log after allowlist restored" + fi + + dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" + dael_expect_access "${repo} allowed after allowlist restored" "200" || return 1 + return 0 +} + +dael_test_cross_repo_denied() { + local src_container="$1" + local keytab="$2" + local principal="$3" + local wrong_repo="$4" + local app_id="$5" + local label="$6" + + dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${wrong_repo}" "${app_id}" + dael_expect_access "${label}" "403" +} + +dael_filter_profiles() { + local want="$1" + local profile + for profile in "${DAEL_PLUGIN_PROFILES[@]}"; do + IFS='|' read -r _ app_id _ _ _ _ <<< "${profile}" + if [[ -z "${want}" || ",${want}," == *",${app_id},"* || ",${want}," == *",$(echo "${profile}" | cut -d'|' -f1),"* ]]; then + echo "${profile}" + fi + done +} diff --git a/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh new file mode 100755 index 00000000000..638f6187edd --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh @@ -0,0 +1,346 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# E2E helpers: dynamic partition-plan POST /plugins (mandatory services) + Kafka partition verification. +# Requires partition-plan-e2e-lib.sh and dynamic-auth-to-local-e2e-lib.sh. + +# repo|pluginId|shortName|partitionCount|source_container|keytab|principal +# partitionCount is the default for POST /plugins when not overridden. +readonly -a DPP_PLUGIN_ONBOARD_SPECS=( + "dev_hdfs|hdfs|hdfs|2|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" + "dev_hive|hiveServer2|hive|2|ranger-hive|/etc/keytabs/hive.keytab|hive/ranger-hive.rangernw@EXAMPLE.COM" + "dev_hbase|hbaseMaster|hbase|2|ranger-hbase|/etc/keytabs/hbase.keytab|hbase/ranger-hbase.rangernw@EXAMPLE.COM" + "dev_kafka|kafka|kafka|2|ranger-kafka|/etc/keytabs/kafka.keytab|kafka/ranger-kafka.rangernw@EXAMPLE.COM" + "dev_knox|knox|knox|2|ranger-knox|/etc/keytabs/knox.keytab|knox/ranger-knox.rangernw@EXAMPLE.COM" + "dev_kms|kms|rangerkms|2|ranger-kms|/etc/keytabs/rangerkms.keytab|rangerkms/ranger-kms.rangernw@EXAMPLE.COM" + "dev_ozone|ozone|om|2|ozone-om|/etc/keytabs/om.keytab|om/ranger-ozone.rangernw@EXAMPLE.COM" +) + +dpp_filter_specs() { + local want="$1" + local spec repo plugin_id + for spec in "${DPP_PLUGIN_ONBOARD_SPECS[@]}"; do + IFS='|' read -r repo plugin_id _ _ _ _ _ <<< "${spec}" + if [[ -z "${want}" || ",${want}," == *",${plugin_id},"* || ",${want}," == *",${repo},"* ]]; then + echo "${spec}" + fi + done +} + +dpp_plan_version() { + local plan_url + plan_url="$(pp_ingestor_plan_url "${CONTAINER}")" + pp_ingestor_request "${CONTAINER}" GET "${plan_url}" + if [[ "${HTTP_CODE}" != "200" ]]; then + echo "" + return 1 + fi + pp_json_field "${HTTP_BODY}" version +} + +dpp_plan_has_plugin() { + local plugin_id="$1" + local plan_json="$2" + printf '%s' "${plan_json}" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); sys.exit(0 if sys.argv[1] in (d.get('plugins') or {}) else 1)" \ + "${plugin_id}" 2>/dev/null +} + +dpp_plan_plugin_partitions() { + local plugin_id="$1" + local plan_json="$2" + printf '%s' "${plan_json}" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); p=(d.get('plugins') or {}).get(sys.argv[1]); \ +print(','.join(str(x) for x in (p or {}).get('partitions') or []))" "${plugin_id}" 2>/dev/null || echo "" +} + +dpp_allowed_users_for_repo() { + local repo="$1" + local short_name="$2" + if [[ "${repo}" == "dev_ozone" ]]; then + printf '%s' "om,ozone" + elif [[ "${repo}" == "dev_hdfs" ]]; then + printf '%s' "hdfs,nn" + else + printf '%s' "${short_name}" + fi +} + +dpp_plan_service_has_users() { + local repo="$1" + local users_csv="$2" + local plan_json="$3" + printf '%s' "${plan_json}" | python3 -c \ + 'import sys,json; d=json.load(sys.stdin); repo=sys.argv[1]; want=set(u.strip() for u in sys.argv[2].split(",") if u.strip()); \ +entry=(d.get("services") or {}).get(repo) or {}; have=set(entry.get("allowedUsers") or []); sys.exit(0 if want.issubset(have) else 1)' \ + "${repo}" "${users_csv}" 2>/dev/null +} + +dpp_onboard_repo_multi() { + local repo="$1" + local plugin_id="$2" + local partition_count="$3" + local users_csv="$4" + local version="$5" + local url body + + url="$(pp_ingestor_plan_url "${CONTAINER}")/plugins" + body="$(pp_build_onboard_plugin_json "${plugin_id}" "${partition_count}" "${version}" "${repo}:${users_csv}")" + pp_ingestor_request "${CONTAINER}" POST "${url}" "${body}" + [[ "${HTTP_CODE}" == "200" ]] +} + +dpp_ensure_plugin_onboarded() { + local spec="$1" + local repo plugin_id short_name part_count container _keytab _principal + local version plan_url users_csv + + IFS='|' read -r repo plugin_id short_name part_count container _keytab _principal <<< "${spec}" + + if ! dael_container_running "${container}"; then + echo " SKIP onboard ${plugin_id}: ${container} not running" + return 0 + fi + + plan_url="$(pp_ingestor_plan_url "${CONTAINER}")" + pp_ingestor_request "${CONTAINER}" GET "${plan_url}" + if [[ "${HTTP_CODE}" != "200" ]]; then + pp_record_fail "GET plan before onboard ${plugin_id}: ${HTTP_CODE}" + return 1 + fi + + users_csv="$(dpp_allowed_users_for_repo "${repo}" "${short_name}")" + + if dpp_plan_has_plugin "${plugin_id}" "${HTTP_BODY}"; then + if dpp_plan_service_has_users "${repo}" "${users_csv}" "${HTTP_BODY}"; then + pp_record_pass "${plugin_id} already in partition plan (allowlist OK)" + return 0 + fi + pp_record_fail "${plugin_id} in plan but allowlist missing [${users_csv}] — rerun with --fresh-plan" + return 1 + fi + + version="$(pp_json_field "${HTTP_BODY}" version)" + + echo " Onboard ${repo} pluginId=${plugin_id} partitions=${part_count} allowedUsers=[${users_csv}]..." + if ! dpp_onboard_repo_multi "${repo}" "${plugin_id}" "${part_count}" "${users_csv}" "${version}"; then + pp_record_fail "POST /plugins ${repo} failed: HTTP ${HTTP_CODE} ${HTTP_BODY}" + return 1 + fi + + pp_record_pass "POST /plugins ${plugin_id} -> plan v$(pp_json_field "${HTTP_BODY}" version)" + if dael_wait_auth_to_local_applied "${CONTAINER}" 45; then + pp_record_pass "auth_to_local recomposed after onboard ${plugin_id}" + else + pp_record_fail "no auth_to_local recompose log after onboard ${plugin_id}" + fi + return 0 +} + +dpp_audit_event_with_marker() { + local repo="$1" + local app_id="$2" + local marker="$3" + local evt_ms + evt_ms="$(python3 -c 'import time; print(int(time.time()*1000))')" + printf '[{"repo":"%s","reqUser":"e2e-partition","evtTime":%s,"access":"read","resource":"/e2e/%s","result":1,"agent":"%s"}]' \ + "${repo}" "${evt_ms}" "${marker}" "${app_id}" +} + +dpp_post_access_with_marker() { + local src_container="$1" + local keytab="$2" + local principal="$3" + local repo="$4" + local app_id="$5" + local marker="$6" + local body url outfile krb_cc body_file + + body="$(dpp_audit_event_with_marker "${repo}" "${app_id}" "${marker}")" + url="$(dael_ingestor_access_url)?serviceName=${repo}&appId=${app_id}" + outfile="/tmp/dpp-access-body-${src_container}-$$" + krb_cc="/tmp/dpp-access-krb-${src_container}-$$" + body_file="/tmp/dpp-access-req-${src_container}-$$" + + printf '%s' "${body}" | docker exec -i "${src_container}" tee "${body_file}" >/dev/null + DAEL_ACCESS_CODE="$(docker exec "${src_container}" bash -c \ + "export KRB5CCNAME=${krb_cc}; kinit -kt '${keytab}' '${principal}' >/dev/null 2>&1 && \ +curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X POST \ + -H 'Content-Type: application/json' -d @${body_file} '${url}'")" + DAEL_ACCESS_BODY="$(docker exec "${src_container}" cat "${outfile}" 2>/dev/null || true)" + docker exec "${src_container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true +} + +dpp_kafka_find_partition_for_marker() { + local record_key="$1" + local marker="$2" + local partitions_csv="${3:-}" + local max_messages="${4:-800}" + local out found partition_id end_offset start_offset + + if [[ -n "${partitions_csv}" ]]; then + for partition_id in $(printf '%s' "${partitions_csv}" | tr ',' ' '); do + [[ -n "${partition_id}" ]] || continue + end_offset="$(pp_kafka_run "/opt/kafka/bin/kafka-get-offsets.sh \ + --bootstrap-server ranger-kafka.rangernw:9092 --topic '${AUDIT_TOPIC}' --partitions ${partition_id} --time -1" \ + | awk -F: '{print $NF}' | tail -1)" + [[ -n "${end_offset}" && "${end_offset}" =~ ^[0-9]+$ ]] || continue + [[ "${end_offset}" -gt 0 ]] || continue + start_offset=$((end_offset > 120 ? end_offset - 120 : 0)) + out="$(pp_kafka_consumer_run "timeout 25 /opt/kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server ranger-kafka.rangernw:9092 \ + --topic '${AUDIT_TOPIC}' \ + --partition ${partition_id} \ + --offset ${start_offset} \ + --max-messages 120 \ + --property print.key=true \ + --timeout-ms 15000")" + found="$(printf '%s' "${out}" | python3 -c " +import sys +key, marker, pid = sys.argv[1:4] +for line in sys.stdin: + if key in line and marker in line: + print(pid) + break +" "${record_key}" "${marker}" "${partition_id}" 2>/dev/null || true)" + if [[ -n "${found}" ]]; then + echo "${found}" + return 0 + fi + done + return 1 + fi + + out="$(pp_kafka_consumer_run "timeout 25 /opt/kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server ranger-kafka.rangernw:9092 \ + --topic '${AUDIT_TOPIC}' \ + --from-beginning \ + --max-messages ${max_messages} \ + --property print.partition=true \ + --property print.key=true \ + --timeout-ms 15000")" + + printf '%s' "${out}" | python3 -c " +import re +import sys +key, marker = sys.argv[1:3] +for line in sys.stdin: + if key not in line or marker not in line: + continue + match = re.search(r'Partition[:\\s]+(\\d+)', line, re.I) + if match: + print(match.group(1)) + break +" "${record_key}" "${marker}" 2>/dev/null || true +} + +dpp_verify_plugin_partition_routing() { + local spec="$1" + local repo plugin_id short_name _part_count container keytab principal + local marker assigned part found + + IFS='|' read -r repo plugin_id short_name _part_count container keytab principal <<< "${spec}" + + if ! dael_container_running "${container}"; then + return 0 + fi + + pp_ingestor_request "${CONTAINER}" GET "$(pp_ingestor_plan_url "${CONTAINER}")" + if [[ "${HTTP_CODE}" != "200" ]]; then + pp_record_fail "GET plan for ${plugin_id} routing check" + return 1 + fi + + assigned="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" + if [[ -z "${assigned}" ]]; then + pp_record_fail "${plugin_id} not in plan — run onboard first" + return 1 + fi + pp_record_pass "${plugin_id} assigned partitions [${assigned}]" + + marker="partition-e2e-${plugin_id}-$(date +%s)" + dpp_post_access_with_marker "${container}" "${keytab}" "${principal}" "${repo}" "${plugin_id}" "${marker}" + if [[ "${DAEL_ACCESS_CODE}" != "200" && "${DAEL_ACCESS_CODE}" != "202" ]]; then + pp_record_fail "${plugin_id} POST /access for partition test: HTTP ${DAEL_ACCESS_CODE} ${DAEL_ACCESS_BODY}" + return 1 + fi + pp_record_pass "${plugin_id} POST /access HTTP ${DAEL_ACCESS_CODE}" + + found="" + for _attempt in 1 2 3 4 5; do + sleep 4 + found="$(dpp_kafka_find_partition_for_marker "${plugin_id}" "${marker}" "${assigned}")" + [[ -n "${found}" ]] && break + done + if [[ -z "${found}" ]]; then + pp_record_fail "${plugin_id} audit not found on Kafka (key=${plugin_id}, marker=${marker}) — check ACLs or dispatcher lag" + return 1 + fi + + if printf '%s' "${assigned}" | python3 -c "import sys; allowed=set(sys.argv[1].split(',')); sys.exit(0 if sys.argv[2] in allowed else 1)" "${assigned}" "${found}"; then + pp_record_pass "${plugin_id} Kafka partition ${found} in assigned [${assigned}]" + return 0 + fi + pp_record_fail "${plugin_id} Kafka partition ${found} NOT in assigned [${assigned}]" + return 1 +} + +dpp_run_harness_plugin_smoke() { + local script="$1" + local label="$2" + if [[ -x "${script}" ]]; then + echo "" + echo " Harness smoke: ${label}..." + if "${script}" 2>/dev/null; then + pp_record_pass "harness ${label} smoke" + else + pp_record_fail "harness ${label} smoke" + fi + fi +} + +dpp_optional_harness_triggers() { + local docker_dir="$1" + dpp_run_harness_plugin_smoke "${docker_dir}/scripts/kms/trigger-kms-plugin-audit-e2e.sh" "KMS" + dpp_run_harness_plugin_smoke "${docker_dir}/scripts/kafka/trigger-kafka-plugin-audit-e2e.sh" "Kafka" + dpp_run_harness_plugin_smoke "${docker_dir}/scripts/knox/trigger-knox-plugin-audit-e2e.sh" "Knox" + dpp_run_harness_plugin_smoke "${docker_dir}/scripts/hbase/trigger-hbase-plugin-audit-e2e.sh" "HBase" +} + +# Solr dispatcher Kerberos can break after ingestor/topic restarts; restart before HDFS pipeline smoke. +dpp_ensure_solr_dispatcher_ready() { + local container="ranger-audit-dispatcher-solr" + local site="/opt/ranger/audit-dispatcher/conf/ranger-audit-dispatcher-solr-site.xml" + local keytab="/etc/keytabs/rangerauditserver.keytab" + local principal="rangerauditserver/ranger-audit-dispatcher-solr.rangernw@EXAMPLE.COM" + + if ! docker ps --filter "name=^${container}$" --filter status=running -q | grep -q .; then + pp_record_fail "Solr dispatcher not running" + return 1 + fi + if docker exec "${container}" test -f "${site}" 2>/dev/null; then + pp_patch_site_prop "${container}" "${site}" "xasecure.audit.jaas.Client.option.useTicketCache" "false" || true + fi + echo " restarting ${container} (Solr Kerberos + consumer reset)..." + docker restart "${container}" >/dev/null + sleep 30 + if docker exec "${container}" bash -c "kinit -kt '${keytab}' '${principal}'" >/dev/null 2>&1 \ + && curl -sf "http://localhost:7091/api/health/ping" >/dev/null 2>&1; then + return 0 + fi + pp_record_fail "Solr dispatcher not healthy after restart" + return 1 +} diff --git a/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh new file mode 100755 index 00000000000..4ef44502c8f --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh @@ -0,0 +1,624 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared helpers for partition-plan E2E scripts. + +PP_E2E_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# shellcheck source=audit-stack-lib.sh +source "${PP_E2E_LIB_DIR}/scripts/audit/audit-stack-lib.sh" + +CONTAINER="${AUDIT_INGESTOR_CONTAINER:-ranger-audit-ingestor}" +REPLICA_CONTAINER="${AUDIT_INGESTOR_REPLICA_CONTAINER:-ranger-audit-ingestor-2}" +KAFKA_CONTAINER="${KAFKA_CONTAINER:-ranger-kafka}" +PROP_DYNAMIC="ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled" +PLAN_TOPIC="ranger_audit_partition_plan" +AUDIT_TOPIC="ranger_audits" +INGESTOR_PLAN_PATH="/api/audit/partition-plan" +KEYTAB="/etc/keytabs/HTTP.keytab" +HTTP_PRINCIPAL="HTTP/ranger-audit-ingestor.rangernw@EXAMPLE.COM" +INGESTOR_HTTP_PORT="7081" +KAFKA_KEYTAB="/etc/keytabs/kafka.keytab" +KAFKA_PRINCIPAL="kafka/ranger-kafka.rangernw@EXAMPLE.COM" +SITE_XMLS=( + "/opt/ranger/audit-ingestor/conf/ranger-audit-ingestor-site.xml" + "/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" +) + +# Example plugin list for E2E promote tests when site XML leaves configured.plugins empty. +PP_CONFIGURED_PLUGINS="hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi" +PP_BUFFER_PROMOTE_CANDIDATES=(storm ambari atlas impala sqoop kylin presto elasticsearch) +PP_DEFAULT_PROMOTE_PLUGIN="storm" + +PP_PASS=0 +PP_FAIL=0 +HTTP_CODE="" +HTTP_BODY="" + +pp_record_pass() { + PP_PASS=$((PP_PASS + 1)) + echo " PASS: $*" +} + +pp_record_fail() { + PP_FAIL=$((PP_FAIL + 1)) + echo " FAIL: $*" >&2 +} + +pp_require_container() { + local name="$1" + if ! docker ps --filter "name=^${name}$" --filter status=running --format '{{.Names}}' | grep -qx "${name}"; then + echo "ERROR: container ${name} is not running" >&2 + exit 1 + fi +} + +pp_json_field() { + local json="$1" + local field="$2" + printf '%s' "${json}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('${field}',''))" 2>/dev/null || echo "" +} + +# Build POST /partition-plan/plugins JSON. services_spec: "repo1:user1,user2|repo2:user3" +pp_build_onboard_plugin_json() { + local plugin_id="$1" + local partition_count="$2" + local expected_version="$3" + local services_spec="$4" + python3 - "${plugin_id}" "${partition_count}" "${expected_version}" "${services_spec}" <<'PY' +import json, sys +plugin_id, part, ver, spec = sys.argv[1:5] +services = {} +for entry in spec.split("|"): + entry = entry.strip() + if not entry: + continue + repo, users = entry.split(":", 1) + services[repo.strip()] = {"allowedUsers": [u.strip() for u in users.split(",") if u.strip()]} +print(json.dumps({ + "pluginId": plugin_id, + "partitionCount": int(part), + "expectedVersion": int(ver), + "services": services, +})) +PY +} + +pp_read_site_prop() { + local container="$1" + local prop="$2" + local site path + for site in "${SITE_XMLS[@]}"; do + if docker exec "${container}" test -f "${site}" 2>/dev/null; then + path="${site}" + break + fi + done + [[ -n "${path:-}" ]] || return 1 + docker exec "${container}" python3 -c \ + "import xml.etree.ElementTree as ET; root=ET.parse('${path}').getroot(); want='${prop}'; \ +[print((v.text or '').strip()) for p in root.findall('property') for n in [p.find('name')] if n is not None and (n.text or '').strip()==want for v in [p.find('value')] if v is not None]" \ + 2>/dev/null || true +} + +pp_expected_topic_partitions() { + local container="$1" + local plugins per buffer count + plugins="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" + per="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin")" + buffer="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions.buffer")" + per="${per:-3}" + buffer="${buffer:-9}" + if [[ -z "${plugins}" ]]; then + per="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions")" + echo "${per:-10}" + return 0 + fi + count="$(printf '%s' "${plugins}" | python3 -c "import sys; print(len([p for p in sys.stdin.read().split(',') if p.strip()]))")" + echo $(( count * per + buffer )) +} + +pp_pick_buffer_promote_plugin() { + local container="$1" + local plan_url="$2" + local configured="${3:-}" + local candidate + + if [[ -z "${configured}" ]]; then + configured="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" + fi + if [[ -z "${configured}" ]]; then + configured="${PP_CONFIGURED_PLUGINS}" + fi + + for candidate in "${PP_BUFFER_PROMOTE_CANDIDATES[@]}"; do + if [[ ",${configured}," == *",${candidate},"* ]]; then + continue + fi + pp_ingestor_request "${container}" GET "${plan_url}" + if [[ "${HTTP_CODE}" != "200" ]]; then + continue + fi + if ! printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if sys.argv[1] in (d.get('plugins') or {}) else 1)" "${candidate}" 2>/dev/null; then + echo "${candidate}" + return 0 + fi + done + # Prior test runs may have promoted all named candidates; use a unique buffer plugin id. + local synthetic="e2eBuffer$(date +%s)" + echo "${synthetic}" +} + +pp_ingestor_host() { + local container="$1" + if [[ "${container}" == "${REPLICA_CONTAINER}" ]]; then + echo "${REPLICA_CONTAINER}.rangernw" + else + echo "${CONTAINER}.rangernw" + fi +} + +# SPNEGO requires the HTTP Host to match the keytab principal (FQDN, not localhost). +pp_ingestor_plan_url() { + local container="$1" + echo "http://$(pp_ingestor_host "${container}"):${INGESTOR_HTTP_PORT}${INGESTOR_PLAN_PATH}" +} + +pp_http_principal_for() { + local container="$1" + if [[ "${container}" == "${REPLICA_CONTAINER}" ]]; then + echo "HTTP/${REPLICA_CONTAINER}.rangernw@EXAMPLE.COM" + else + echo "${HTTP_PRINCIPAL}" + fi +} + +pp_ingestor_request() { + local container="${1:?}" + local method="${2:?}" + local url="${3:?}" + local body="${4:-}" + local outfile="/tmp/pp-e2e-body-${container}-$$" + local krb_cc="/tmp/pp-e2e-krb-${container}-$$" + local body_file="" + local curl_cmd + local principal + principal="$(pp_http_principal_for "${container}")" + + curl_cmd="export KRB5CCNAME=${krb_cc}; kinit -kt ${KEYTAB} ${principal} >/dev/null 2>&1" + if [[ -n "${body}" ]]; then + body_file="/tmp/pp-e2e-req-${container}-$$" + printf '%s' "${body}" | docker exec -i "${container}" tee "${body_file}" >/dev/null + curl_cmd+="; curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X ${method}" + curl_cmd+=" -H 'Content-Type: application/json' -d @${body_file} '${url}'" + else + curl_cmd+="; curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X ${method} '${url}'" + fi + + HTTP_CODE="$(docker exec "${container}" bash -c "${curl_cmd}")" + HTTP_BODY="$(docker exec "${container}" cat "${outfile}" 2>/dev/null || true)" + docker exec "${container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true +} + +pp_dynamic_enabled() { + local container="$1" + local path="$2" + docker exec "${container}" grep -A1 "${PROP_DYNAMIC}" "${path}" 2>/dev/null | grep -qi 'true' +} + +pp_patch_site_prop() { + local container="$1" + local site="$2" + local prop="$3" + local value="$4" + docker exec -i -u root "${container}" python3 - "${site}" "${prop}" "${value}" <<'PY' +import sys +import xml.etree.ElementTree as ET + +path, prop, value = sys.argv[1:4] +root = ET.parse(path).getroot() +found = False +for p in root.findall("property"): + name = p.find("name") + if name is None or (name.text or "").strip() != prop: + continue + val = p.find("value") + if val is None: + val = ET.SubElement(p, "value") + val.text = value + found = True + break +if not found: + prop_el = ET.SubElement(root, "property") + ET.SubElement(prop_el, "name").text = prop + ET.SubElement(prop_el, "value").text = value +ET.indent(root, space=" ") +tree = ET.ElementTree(root) +tree.write(path, encoding="unicode", xml_declaration=False) +PY +} + +pp_ensure_audit_partitioner_for_dynamic() { + local container="${1:-${CONTAINER}}" + local seed_plugin="${2:-${PP_DYNAMIC_PRODUCER_PLUGIN:-hdfs}}" + local plugins + plugins="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" + if [[ -n "${plugins}" ]]; then + return 0 + fi + for site in "${SITE_XMLS[@]}"; do + if docker exec "${container}" test -f "${site}" 2>/dev/null; then + pp_patch_site_prop "${container}" "${site}" "ranger.audit.ingestor.kafka.configured.plugins" "${seed_plugin}" + fi + done +} + +pp_set_dynamic_enabled() { + local container="$1" + local value="$2" + local restart="${3:-true}" + local changed=false + for site in "${SITE_XMLS[@]}"; do + if ! docker exec "${container}" test -f "${site}" 2>/dev/null; then + continue + fi + pp_patch_site_prop "${container}" "${site}" "${PROP_DYNAMIC}" "${value}" + changed=true + done + if [[ "${value}" == "true" ]]; then + pp_ensure_audit_partitioner_for_dynamic "${container}" + pp_prepare_audit_topic_for_greenfield "${container}" + fi + if [[ "${changed}" == "true" && "${restart}" == "true" ]]; then + echo " Restarting ${container} (dynamic=${value})..." + PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker restart "${container}" >/dev/null + sleep 45 + fi +} + +pp_wait_health() { + local port="${1:-7081}" + local label="${2:-Ingestor}" + local timeout="${3:-120}" + audit_stack_wait_url "http://localhost:${port}/api/audit/health" "${label}" "${timeout}" +} + +pp_wait_watcher() { + local container="$1" + local timeout="${2:-300}" + local deadline=$((SECONDS + timeout)) + local url log_args=() + + url="$(pp_ingestor_plan_url "${container}")" + if [[ -n "${PP_RESTART_SINCE:-}" ]]; then + log_args=(--since "${PP_RESTART_SINCE}") + fi + + while (( SECONDS < deadline )); do + pp_ingestor_request "${container}" GET "${url}" + if [[ "${HTTP_CODE}" == "200" ]]; then + return 0 + fi + if docker logs ${log_args+"${log_args[@]}"} "${container}" 2>&1 | grep -q "Partition plan watcher ready"; then + return 0 + fi + sleep 5 + done + return 1 +} + +pp_kafka_run() { + local script="$1" + docker exec "${KAFKA_CONTAINER}" bash -c \ + "cat > /tmp/pp-kafka-client.properties <<'EOF' +security.protocol=SASL_PLAINTEXT +sasl.kerberos.service.name=kafka +EOF +cat > /tmp/pp-kafka-client-jaas.conf </dev/null || true +} + +# kafka-console-consumer uses --consumer.config (not --command-config). +pp_kafka_consumer_run() { + local script="$1" + docker exec "${KAFKA_CONTAINER}" bash -c \ + "cat > /tmp/pp-kafka-client.properties <<'EOF' +security.protocol=SASL_PLAINTEXT +sasl.kerberos.service.name=kafka +EOF +cat > /tmp/pp-kafka-client-jaas.conf </dev/null || true +} + + +pp_stop_audit_stack_consumers() { + docker stop ranger-audit-ingestor ranger-audit-dispatcher-solr ranger-audit-dispatcher-hdfs 2>/dev/null || true +} + +pp_start_audit_stack_consumers() { + docker start ranger-audit-ingestor ranger-audit-dispatcher-solr ranger-audit-dispatcher-hdfs 2>/dev/null || true +} + +pp_kafka_wait_topic_absent() { + local topic="$1" + local timeout="${2:-180}" + local deadline=$((SECONDS + timeout)) + while (( SECONDS < deadline )); do + if ! pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${topic}"; then + return 0 + fi + sleep 5 + done + echo "ERROR: Kafka topic '${topic}' still present after ${timeout}s" >&2 + return 1 +} + +pp_kafka_topic_partition_count() { + local topic="$1" + local out + out="$(pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --describe --topic '${topic}'")" + printf '%s' "${out}" | python3 -c "import sys,re; t=sys.stdin.read(); m=re.search(r'PartitionCount:\\s*(\\d+)', t); print(m.group(1) if m else '')" 2>/dev/null || echo "" +} + +pp_delete_plan_topic() { + pp_stop_audit_stack_consumers + pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${PLAN_TOPIC}'" + pp_kafka_wait_topic_absent "${PLAN_TOPIC}" 180 || true + pp_start_audit_stack_consumers + sleep 10 +} + +# Greenfield bootstrap requires plan.topicPartitionCount == Kafka ranger_audits partitions. +pp_prepare_audit_topic_for_greenfield() { + local container="${1:-${CONTAINER}}" + local expected actual + + expected="$(pp_expected_topic_partitions "${container}")" + actual="$(pp_kafka_topic_partition_count "${AUDIT_TOPIC}")" + if [[ -z "${actual}" || "${actual}" == "${expected}" ]]; then + return 0 + fi + echo " Recreating ${AUDIT_TOPIC} (${actual} partitions -> ${expected} for site XML layout)..." + pp_stop_audit_stack_consumers + pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${AUDIT_TOPIC}'" + pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${PLAN_TOPIC}'" || true + pp_kafka_wait_topic_absent "${AUDIT_TOPIC}" 180 + pp_kafka_wait_topic_absent "${PLAN_TOPIC}" 180 || true + pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --create --topic '${AUDIT_TOPIC}' --partitions ${expected} --replication-factor 1" + sleep 3 + pp_start_audit_stack_consumers + sleep 10 +} + +pp_seed_plan_json() { + local json="$1" + local tmp + tmp="$(mktemp)" + printf '%s' "${json}" > "${tmp}" + pp_seed_plan_file "${tmp}" + rm -f "${tmp}" +} + +pp_seed_plan_file() { + local file="$1" + pp_ensure_plan_topic + { printf '%s:' "${AUDIT_TOPIC}"; cat "${file}"; } | docker exec -i "${KAFKA_CONTAINER}" bash -c \ + "cat > /tmp/pp-kafka-client.properties <<'EOF' +security.protocol=SASL_PLAINTEXT +sasl.kerberos.service.name=kafka +EOF +cat > /tmp/pp-kafka-client-jaas.conf </dev/null 2>&1 || true + sleep 2 +} + +pp_ensure_plan_topic() { + if ! pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${PLAN_TOPIC}"; then + pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --create --topic '${PLAN_TOPIC}' --partitions 1 --replication-factor 1 --config cleanup.policy=compact" + fi +} + +pp_ensure_replica_keytab() { + local rep_fqdn="${REPLICA_CONTAINER}.rangernw" + local host_kt="${PP_E2E_LIB_DIR}/dist/keytabs/${REPLICA_CONTAINER}" + docker exec ranger-kdc bash -c " + set -euo pipefail + keytab_dir=/etc/keytabs/${REPLICA_CONTAINER} + mkdir -p \"\${keytab_dir}\" + kadmin.local -q 'addprinc -randkey HTTP/${rep_fqdn}' >/dev/null 2>&1 || true + kadmin.local -q 'addprinc -randkey rangerauditserver/${rep_fqdn}' >/dev/null 2>&1 || true + rm -f \"\${keytab_dir}/HTTP.keytab\" \"\${keytab_dir}/rangerauditserver.keytab\" + kadmin.local -q 'ktadd -k '\${keytab_dir}'/HTTP.keytab HTTP/${rep_fqdn}' + kadmin.local -q 'ktadd -k '\${keytab_dir}'/rangerauditserver.keytab rangerauditserver/${rep_fqdn}' + chmod 444 \"\${keytab_dir}/\"*.keytab + " + mkdir -p "${host_kt}" + docker cp "ranger-kdc:/etc/keytabs/${REPLICA_CONTAINER}/." "${host_kt}/" >/dev/null +} + +pp_patch_replica_site_file() { + local file="$1" + local rep_fqdn="${REPLICA_CONTAINER}.rangernw" + local primary_fqdn="${CONTAINER}.rangernw" + python3 - "${file}" "${primary_fqdn}" "${rep_fqdn}" <<'PY' +import sys +import xml.etree.ElementTree as ET + +path, primary_fqdn, rep_fqdn = sys.argv[1:4] +props = { + "ranger.audit.ingestor.host": rep_fqdn, + "ranger.audit.ingestor.bind.address": rep_fqdn, +} +root = ET.parse(path).getroot() +for prop in root.findall("property"): + name = prop.find("name") + if name is None: + continue + key = (name.text or "").strip() + if key in props: + value = prop.find("value") + if value is None: + value = ET.SubElement(prop, "value") + value.text = props[key] +ET.indent(root, space=" ") +tree = ET.ElementTree(root) +tree.write(path, encoding="unicode", xml_declaration=False) +PY +} + +pp_replica_config_staging() { + echo "${PP_E2E_LIB_DIR}/dist/${REPLICA_CONTAINER}-conf" +} + +pp_sync_replica_config_from_primary() { + local preserve_mount="${1:-false}" + local staging + staging="$(pp_replica_config_staging)" + if [[ "${preserve_mount}" == "true" ]]; then + mkdir -p "${staging}/conf" "${staging}/webapp-conf" + else + rm -rf "${staging}" + mkdir -p "${staging}/conf" "${staging}/webapp-conf" + fi + docker cp "${CONTAINER}:/opt/ranger/audit-ingestor/conf/." "${staging}/conf/" >/dev/null + if docker exec "${CONTAINER}" test -f "${SITE_XMLS[1]}" 2>/dev/null; then + docker cp "${CONTAINER}:${SITE_XMLS[1]}" "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" >/dev/null + else + cp "${staging}/conf/ranger-audit-ingestor-site.xml" "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" + fi + pp_patch_replica_site_file "${staging}/conf/ranger-audit-ingestor-site.xml" + pp_patch_replica_site_file "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" +} + +pp_start_ingestor_replica() { + local image="${1:-ranger-audit-ingestor:latest}" + local staging + if docker ps -a --format '{{.Names}}' | grep -qx "${REPLICA_CONTAINER}"; then + docker rm -f "${REPLICA_CONTAINER}" >/dev/null 2>&1 || true + fi + pp_ensure_replica_keytab + pp_sync_replica_config_from_primary + staging="$(pp_replica_config_staging)" + if ! docker run -d --name "${REPLICA_CONTAINER}" \ + --network rangernw \ + --hostname "${REPLICA_CONTAINER}.rangernw" \ + -p 7082:7081 \ + -e JAVA_HOME=/opt/java/openjdk \ + -e AUDIT_INGESTOR_HOME_DIR=/opt/ranger/audit-ingestor \ + -e AUDIT_INGESTOR_CONF_DIR=/opt/ranger/audit-ingestor/conf \ + -e AUDIT_INGESTOR_LOG_DIR=/var/log/ranger/audit-ingestor \ + -e AUDIT_INGESTOR_HEAP="-Xms512m -Xmx2g" \ + -e KERBEROS_ENABLED=true \ + -e KAFKA_BOOTSTRAP_SERVERS=ranger-kafka.rangernw:9092 \ + -v "${PP_E2E_LIB_DIR}/dist/keytabs/${REPLICA_CONTAINER}:/etc/keytabs" \ + -v "${staging}/conf:/opt/ranger/audit-ingestor/conf:ro" \ + "${image}" >/dev/null; then + echo "ERROR: docker run failed for ${REPLICA_CONTAINER}" >&2 + return 1 + fi + PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + sleep 60 + if ! pp_wait_health 7082 "Ingestor replica" 300; then + return 1 + fi + pp_install_replica_webapp_site +} + +pp_install_replica_webapp_site() { + local staging + staging="$(pp_replica_config_staging)" + if ! docker exec "${REPLICA_CONTAINER}" test -d \ + /opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf 2>/dev/null; then + return 0 + fi + docker cp "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" \ + "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" >/dev/null + PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker restart "${REPLICA_CONTAINER}" >/dev/null + sleep 60 + pp_wait_health 7082 "Ingestor replica after webapp site sync" 300 +} + +pp_stop_ingestor_replica() { + docker rm -f "${REPLICA_CONTAINER}" >/dev/null 2>&1 || true +} + +pp_copy_site_to_replica() { + local staging restart="${2:-true}" + pp_sync_replica_config_from_primary "true" + staging="$(pp_replica_config_staging)" + docker cp "${staging}/conf/ranger-audit-ingestor-site.xml" \ + "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/conf/ranger-audit-ingestor-site.xml" + if docker exec "${REPLICA_CONTAINER}" test -d \ + /opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf 2>/dev/null; then + docker cp "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" \ + "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" + fi + if [[ "${restart}" == "true" ]]; then + PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker restart "${REPLICA_CONTAINER}" >/dev/null + sleep 45 + pp_wait_health 7082 "Ingestor replica after config copy" 300 + fi +} + +pp_print_results() { + echo "" + echo "Results: ${PP_PASS} passed, ${PP_FAIL} failed" + if [[ "${PP_FAIL}" -ne 0 ]]; then + echo "See: audit-server/README-KAFKA-PARTITION-PLAN-E2E-TEST-PLAN.md" >&2 + return 1 + fi + return 0 +} + +pp_preflight_tier3() { + chmod +x "${PP_E2E_LIB_DIR}/scripts/audit/wait-for-audit-health.sh" 2>/dev/null || true + "${PP_E2E_LIB_DIR}/scripts/audit/wait-for-audit-health.sh" --tier 3 --timeout "${1:-120}" + pp_require_container "${CONTAINER}" + pp_require_container "${KAFKA_CONTAINER}" +} diff --git a/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh new file mode 100755 index 00000000000..f3bc77ee382 --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# E2E: dynamic partition allocation — POST /partition-plan/plugins per plugin + Kafka routing. +# +# Flow (mirrors ranger-audit-e2e-harness plugin matrix): +# 1. Enable dynamic mode (empty configured.plugins → buffer-only bootstrap) +# 2. For each running plugin container: POST /partition-plan/plugins +# (pluginId, partitionCount, expectedVersion, mandatory services map) +# 3. POST /api/audit/access from plugin keytab → verify Kafka record partition ∈ plan +# 4. Optional: run harness trigger-* scripts when present (KMS, Kafka, Knox, HBase) +# +# Prerequisites: Tier 3 Docker (see audit-server/README-DYNAMIC-PARTITION-PLUGIN-E2E.md). +# Reference harness branch: ranger-audit-e2e-harness (plugin verify/trigger scripts). +# +# Usage: +# cd dev-support/ranger-docker +# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh +# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh --no-enable --plugins hdfs,kms +# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh --with-harness-triggers + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${SCRIPT_DIR}" + +# shellcheck source=partition-plan-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" +# shellcheck source=dynamic-auth-to-local-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/dynamic-auth-to-local-e2e-lib.sh" +# shellcheck source=dynamic-partition-plugin-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/dynamic-partition-plugin-e2e-lib.sh" + +DO_ENABLE=true +PLUGIN_FILTER="" +WITH_HARNESS=false +TIMEOUT=300 +FRESH_PLAN=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-enable) DO_ENABLE=false; shift ;; + --plugins) PLUGIN_FILTER="${2:?}"; shift 2 ;; + --with-harness-triggers) WITH_HARNESS=true; shift ;; + --fresh-plan) FRESH_PLAN=true; shift ;; + --timeout) TIMEOUT="${2:?}"; shift 2 ;; + -h|--help) + sed -n '31,36p' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +echo "=== Dynamic partition plugin onboard + routing E2E ===" +pp_preflight_tier3 "${TIMEOUT}" + +if [[ "${DO_ENABLE}" == "true" ]]; then + if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then + echo "Preparing audit topic for greenfield layout..." + pp_prepare_audit_topic_for_greenfield "${CONTAINER}" + if [[ "${FRESH_PLAN}" == "true" ]]; then + pp_delete_plan_topic + fi + echo "Enabling dynamic partition plan..." + pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; pp_print_results; exit 1; } + pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health"; pp_print_results; exit 1; } + fi +fi + +if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then + pp_record_fail "dynamic.enabled must be true" + pp_print_results + exit 1 +fi + +if ! pp_wait_watcher "${CONTAINER}" "${TIMEOUT}"; then + pp_record_fail "PartitionPlanWatcher not ready" + pp_print_results + exit 1 +fi +pp_record_pass "PartitionPlanWatcher ready" + +PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" +pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" +if [[ "${HTTP_CODE}" == "200" ]]; then + pp_record_pass "GET partition-plan returns 200 (v$(pp_json_field "${HTTP_BODY}" version))" +else + pp_record_fail "GET partition-plan expected 200, got ${HTTP_CODE}" + pp_print_results + exit 1 +fi + +echo "" +echo "=== Phase 1: POST /plugins per running plugin (REST, mandatory services) ===" +spec="" +while IFS= read -r spec; do + [[ -n "${spec}" ]] || continue + dpp_ensure_plugin_onboarded "${spec}" || true +done < <(dpp_filter_specs "${PLUGIN_FILTER}") + +echo "" +echo "=== Phase 2: POST /access + Kafka partition ∈ plan ===" +while IFS= read -r spec; do + [[ -n "${spec}" ]] || continue + IFS='|' read -r repo plugin_id _ _ container _ _ <<< "${spec}" + echo "" + echo "--- ${plugin_id} (${repo}) ---" + dpp_verify_plugin_partition_routing "${spec}" || true +done < <(dpp_filter_specs "${PLUGIN_FILTER}") + +if [[ "${WITH_HARNESS}" == "true" ]]; then + echo "" + echo "=== Phase 3: optional ranger-audit-e2e-harness plugin triggers ===" + dpp_optional_harness_triggers "${SCRIPT_DIR}" +fi + +echo "" +echo "See: audit-server/README-DYNAMIC-PARTITION-PLUGIN-E2E.md" +echo "Curl cookbook: dist/audit-e2e/access-ingestor-curl-cookbook.sh (run verify-dynamic-auth-to-local-e2e.sh --generate-curl-only)" + +pp_print_results diff --git a/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh new file mode 100755 index 00000000000..a15742abe91 --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# HDFS-focused dynamic partition E2E: +# 1. Enable dynamic mode (greenfield buffer-only plan) +# 2. POST /partition-plan/plugins — dev_hdfs + hdfs plugin + mandatory services allowlist +# 3. Verify POST /access + Kafka partition routing +# 4. PATCH /partition-plan/plugins/hdfs — expand partition count +# 5. Verify routing again after scale +# +# Usage: +# cd dev-support/ranger-docker +# ./scripts/audit/verify-hdfs-dynamic-partition-e2e.sh +# ./scripts/audit/verify-hdfs-dynamic-partition-e2e.sh --with-hdfs-trigger + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${SCRIPT_DIR}" + +# shellcheck source=partition-plan-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" +# shellcheck source=dynamic-auth-to-local-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/dynamic-auth-to-local-e2e-lib.sh" +# shellcheck source=dynamic-partition-plugin-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/dynamic-partition-plugin-e2e-lib.sh" + +readonly HDFS_SPEC="dev_hdfs|hdfs|hdfs|2|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" +SCALE_ADDITIONAL="${SCALE_ADDITIONAL:-2}" +TIMEOUT=300 +FRESH_PLAN=false +WITH_HDFS_TRIGGER=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --fresh-plan) FRESH_PLAN=true; shift ;; + --with-hdfs-trigger) WITH_HDFS_TRIGGER=true; shift ;; + --scale-additional) SCALE_ADDITIONAL="${2:?}"; shift 2 ;; + --timeout) TIMEOUT="${2:?}"; shift 2 ;; + -h|--help) + sed -n '26,29p' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +hdfs_scale_plugin() { + local plugin_id="$1" + local additional="$2" + local version="$3" + local url body before after attempt + + pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" + before="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" + + url="${PLAN_URL}/plugins/${plugin_id}" + body="{\"additionalPartitions\":${additional},\"expectedVersion\":${version}}" + for attempt in 1 2 3; do + echo " PATCH ${url} (+${additional} partitions, expectedVersion=${version}, attempt ${attempt})..." + pp_ingestor_request "${CONTAINER}" PATCH "${url}" "${body}" + if [[ "${HTTP_CODE}" == "200" ]]; then + after="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" + pp_record_pass "scale ${plugin_id} partitions [${before}] -> [${after}] plan v$(pp_json_field "${HTTP_BODY}" version)" + if [[ "$(echo "${before}" | tr ',' '\n' | wc -l | tr -d ' ')" -ge "$(echo "${after}" | tr ',' '\n' | wc -l | tr -d ' ')" ]]; then + pp_record_fail "scale ${plugin_id} did not increase partition count" + return 1 + fi + return 0 + fi + if [[ "${HTTP_CODE}" == "503" && "${attempt}" -lt 3 ]]; then + echo " scale returned 503 (Kafka grow timeout?) — retrying in 15s..." + sleep 15 + version="$(dpp_plan_version)" + body="{\"additionalPartitions\":${additional},\"expectedVersion\":${version}}" + continue + fi + pp_record_fail "scale ${plugin_id} expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" + return 1 + done + return 1 +} + +echo "=== HDFS dynamic partition REST + routing E2E ===" +if [[ -x "${SCRIPT_DIR}/scripts/audit/ensure-audit-ingestor-plugin-users.sh" ]]; then + "${SCRIPT_DIR}/scripts/audit/ensure-audit-ingestor-plugin-users.sh" || true +fi +for site in "${SITE_XMLS[@]}"; do + if docker exec "${CONTAINER}" test -f "${site}" 2>/dev/null; then + pp_patch_site_prop "${CONTAINER}" "${site}" "ranger.audit.ingestor.service.dev_hdfs.allowed.users" "hdfs,nn" + fi +done +pp_preflight_tier3 "${TIMEOUT}" + +if ! dael_container_running "ranger-hadoop"; then + echo "ERROR: ranger-hadoop is not running. Start HDFS stack first, e.g.:" >&2 + echo " ./setup-audit-e2e.sh up --hdfs-only --no-verify" >&2 + exit 1 +fi + +PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" + +echo "" +echo "=== Step 1: Enable dynamic partition plan ===" +if [[ "${FRESH_PLAN}" == "true" ]]; then + echo " Fresh plan requested — resetting plan topic and greenfield audit topic layout..." + pp_set_dynamic_enabled "${CONTAINER}" "false" "false" || true + pp_prepare_audit_topic_for_greenfield "${CONTAINER}" + pp_delete_plan_topic +fi +if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then + pp_prepare_audit_topic_for_greenfield "${CONTAINER}" + if [[ "${FRESH_PLAN}" == "true" ]]; then + pp_delete_plan_topic + fi + pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; pp_print_results; exit 1; } + pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health"; pp_print_results; exit 1; } +else + pp_record_pass "dynamic mode already enabled" +fi + +pp_wait_watcher "${CONTAINER}" "${TIMEOUT}" || { pp_record_fail "watcher"; pp_print_results; exit 1; } +pp_record_pass "PartitionPlanWatcher ready" + +echo "" +echo "=== Step 2: POST /partition-plan/plugins (dev_hdfs + allowlist) ===" +dpp_ensure_plugin_onboarded "${HDFS_SPEC}" || { pp_print_results; exit 1; } + +echo "" +echo "=== Step 3: E2E after onboard — POST /access + Kafka partition routing ===" +dpp_verify_plugin_partition_routing "${HDFS_SPEC}" || true + +if [[ "${WITH_HDFS_TRIGGER}" == "true" && -x "${SCRIPT_DIR}/setup-audit-e2e.sh" ]]; then + echo "" + echo "=== Step 3b: Full HDFS audit pipeline (dfs smoke) ===" + dpp_ensure_solr_dispatcher_ready || true + if "${SCRIPT_DIR}/setup-audit-e2e.sh" trigger-hdfs-audit --no-verify 2>/dev/null; then + pp_record_pass "trigger-hdfs-audit pipeline" + else + pp_record_fail "trigger-hdfs-audit pipeline" + fi +fi + +echo "" +echo "=== Step 4: PATCH /partition-plan/plugins/hdfs (expand partitions) ===" +version="$(dpp_plan_version)" +if [[ -z "${version}" ]]; then + pp_record_fail "could not read plan version before scale" + pp_print_results + exit 1 +fi +hdfs_scale_plugin "hdfs" "${SCALE_ADDITIONAL}" "${version}" || true +dael_wait_auth_to_local_applied "${CONTAINER}" 45 && pp_record_pass "auth_to_local after scale" || pp_record_fail "auth_to_local after scale" +echo " waiting 10s for ingestor partitioner metadata after scale..." +sleep 10 + +echo "" +echo "=== Step 5: E2E after scale — POST /access + Kafka partition routing ===" +dpp_verify_plugin_partition_routing "${HDFS_SPEC}" || true + +if [[ "${WITH_HDFS_TRIGGER}" == "true" && -x "${SCRIPT_DIR}/setup-audit-e2e.sh" ]]; then + echo "" + echo "=== Step 5b: Full HDFS audit pipeline after scale ===" + dpp_ensure_solr_dispatcher_ready || true + if "${SCRIPT_DIR}/setup-audit-e2e.sh" trigger-hdfs-audit --no-verify 2>/dev/null; then + pp_record_pass "trigger-hdfs-audit after scale" + else + pp_record_fail "trigger-hdfs-audit after scale" + fi +fi + +pp_print_results +[[ "${PP_FAIL}" -eq 0 ]] diff --git a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh new file mode 100755 index 00000000000..b5c6911d180 --- /dev/null +++ b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core E2E: static regression + dynamic REST (promote/scale/409/400). +# +# Usage: +# ./scripts/audit/verify-partition-plan-e2e.sh --static-only +# ./scripts/audit/verify-partition-plan-e2e.sh --dynamic --restore-static +# ./scripts/audit/verify-partition-plan-e2e.sh --dynamic --with-audit-smoke +# +# Full suite: ./scripts/audit/verify-partition-plan-e2e-all.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${SCRIPT_DIR}" + +# shellcheck source=partition-plan-e2e-lib.sh +source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" + +RUN_STATIC=false +RUN_DYNAMIC=false +DO_ENABLE=true +RESTORE_STATIC=false +AUDIT_SMOKE=false +TIMEOUT=300 +PROMOTE_PLUGIN="storm" + +while [[ $# -gt 0 ]]; do + case "$1" in + --static-only) RUN_STATIC=true; shift ;; + --dynamic) RUN_DYNAMIC=true; shift ;; + --no-enable) DO_ENABLE=false; shift ;; + --restore-static) RESTORE_STATIC=true; shift ;; + --with-audit-smoke) AUDIT_SMOKE=true; shift ;; + --timeout) TIMEOUT="${2:?}"; shift 2 ;; + --promote-plugin) PROMOTE_PLUGIN="${2:?}"; shift 2 ;; + -h|--help) + sed -n '19,25p' "$0" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +if [[ "${RUN_STATIC}" != "true" && "${RUN_DYNAMIC}" != "true" ]]; then + RUN_STATIC=true + RUN_DYNAMIC=true +fi + +PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" + +run_static_tests() { + echo "" + echo "=== Static mode (dynamic.enabled=false) ===" + if pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then + pp_record_fail "dynamic.enabled should be false (use --restore-static)" + return + fi + pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" + if [[ "${HTTP_CODE}" == "503" ]]; then + pp_record_pass "GET partition-plan returns 503 when disabled" + else + pp_record_fail "GET expected 503, got ${HTTP_CODE}" + fi + if curl -sf "http://localhost:7081/api/audit/health" >/dev/null; then + pp_record_pass "GET /health still 200" + else + pp_record_fail "GET /health not reachable" + fi +} + +run_dynamic_tests() { + echo "" + echo "=== Dynamic mode ===" + if [[ "${DO_ENABLE}" == "true" ]]; then + echo "Preparing greenfield Kafka layout (static mode)..." + pp_set_dynamic_enabled "${CONTAINER}" "false" "false" || true + pp_prepare_audit_topic_for_greenfield "${CONTAINER}" + pp_delete_plan_topic + echo "Enabling dynamic partition plan..." + pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; return; } + pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health after enable"; return; } + fi + if ! pp_wait_watcher "${CONTAINER}" "${TIMEOUT}"; then + pp_record_fail "PartitionPlanWatcher not ready" + return + fi + pp_record_pass "PartitionPlanWatcher ready" + + pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" + if [[ "${HTTP_CODE}" != "200" ]]; then + pp_record_fail "GET expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" + return + fi + pp_record_pass "GET partition-plan returns 200" + + local version topic_count kafka_parts new_version plan_plugins expected_plugin_count promote_plugin + version="$(pp_json_field "${HTTP_BODY}" version)" + if [[ -n "${version}" && "${version}" -ge 1 ]]; then + pp_record_pass "plan version=${version}" + else + pp_record_fail "invalid version: ${version}" + fi + + plan_plugins="$(printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('plugins') or {}))" 2>/dev/null || echo "")" + local configured_plugins + configured_plugins="$(pp_read_site_prop "${CONTAINER}" "ranger.audit.ingestor.kafka.configured.plugins")" + if [[ -n "${configured_plugins}" ]]; then + expected_plugin_count="$(printf '%s' "${configured_plugins}" | python3 -c "import sys; print(len([p for p in sys.stdin.read().split(',') if p.strip()]))")" + else + expected_plugin_count="0" + fi + topic_count="$(pp_json_field "${HTTP_BODY}" topicPartitionCount)" + kafka_parts="$(pp_kafka_topic_partition_count "${AUDIT_TOPIC}")" + if [[ "${version}" == "1" && "${plan_plugins}" == "${expected_plugin_count}" ]]; then + if [[ "${expected_plugin_count}" == "0" ]]; then + pp_record_pass "greenfield plan is buffer-only (configured.plugins empty in site XML)" + else + pp_record_pass "greenfield plan lists ${plan_plugins} configured plugins (site XML layout)" + fi + elif [[ "${version}" != "1" ]]; then + pp_record_pass "plan has ${plan_plugins} plugins at v${version}" + else + pp_record_fail "greenfield expected ${expected_plugin_count} configured plugins, plan has ${plan_plugins}" + fi + if [[ -n "${topic_count}" && "${topic_count}" == "${kafka_parts}" ]]; then + pp_record_pass "topicPartitionCount matches Kafka (${topic_count})" + else + pp_record_fail "topicPartitionCount=${topic_count} kafka=${kafka_parts}" + fi + + local services_count + services_count="$(printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('services') or {}))" 2>/dev/null || echo "0")" + if [[ "${services_count}" -ge 1 ]]; then + pp_record_pass "plan includes services allowlist (${services_count} repos)" + else + pp_record_fail "plan missing services allowlist map" + fi + + promote_plugin="${PROMOTE_PLUGIN}" + if [[ "${PROMOTE_PLUGIN}" == "storm" ]]; then + promote_plugin="$(pp_pick_buffer_promote_plugin "${CONTAINER}" "${PLAN_URL}")" + fi + + if pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${PLAN_TOPIC}"; then + pp_record_pass "plan topic ${PLAN_TOPIC} exists" + else + pp_record_fail "plan topic missing" + fi + + echo "" + echo " Onboard without services (400)..." + pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ + "{\"pluginId\":\"e2eNoServices\",\"partitionCount\":2,\"expectedVersion\":${version}}" + if [[ "${HTTP_CODE}" == "400" ]] && echo "${HTTP_BODY}" | grep -qi "services"; then + pp_record_pass "onboard without services -> 400" + else + pp_record_fail "onboard without services expected 400, got ${HTTP_CODE}: ${HTTP_BODY}" + fi + + echo "" + echo " Onboard ${promote_plugin} (buffer plugin + mandatory services)..." + local onboard_body onboard_repo="dev_${promote_plugin}" + onboard_body="$(pp_build_onboard_plugin_json "${promote_plugin}" 2 "${version}" "${onboard_repo}:${promote_plugin}")" + pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" "${onboard_body}" + if [[ "${HTTP_CODE}" == "200" ]]; then + new_version="$(pp_json_field "${HTTP_BODY}" version)" + if [[ "${new_version}" -gt "${version}" ]] && echo "${HTTP_BODY}" | grep -q "\"${promote_plugin}\""; then + pp_record_pass "onboard ${promote_plugin} -> v${new_version}" + version="${new_version}" + else + pp_record_fail "onboard response invalid" + fi + else + pp_record_fail "onboard expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" + fi + + echo " Stale expectedVersion (409)..." + pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ + "$(pp_build_onboard_plugin_json "e2eStale409" 2 1 "dev_e2eStale409:e2eStale409")" + if [[ "${HTTP_CODE}" == "409" ]]; then + pp_record_pass "stale expectedVersion -> 409" + else + pp_record_fail "stale expectedVersion expected 409, got ${HTTP_CODE}" + fi + + echo " Onboard hdfs again (400)..." + pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ + "$(pp_build_onboard_plugin_json "hdfs" 2 "${version}" "dev_hdfs:hdfs")" + if [[ "${HTTP_CODE}" == "400" ]]; then + pp_record_pass "duplicate hdfs promote -> 400" + else + pp_record_fail "duplicate promote expected 400, got ${HTTP_CODE}" + fi + + echo " Multi-service onboard (trino + dev_trino, dev_trino2)..." + local multi_body + multi_body="$(pp_build_onboard_plugin_json "trino" 2 "${version}" "dev_trino:trino|dev_trino2:trino2")" + pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" "${multi_body}" + if [[ "${HTTP_CODE}" == "200" ]]; then + new_version="$(pp_json_field "${HTTP_BODY}" version)" + if [[ "${new_version}" -gt "${version}" ]] \ + && echo "${HTTP_BODY}" | grep -q '"dev_trino"' \ + && echo "${HTTP_BODY}" | grep -q '"dev_trino2"'; then + pp_record_pass "multi-service onboard trino -> v${new_version}" + version="${new_version}" + else + pp_record_fail "multi-service onboard response invalid" + fi + else + pp_record_fail "multi-service onboard expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" + fi + + echo " Scale ${promote_plugin}..." + pp_ingestor_request "${CONTAINER}" PATCH "${PLAN_URL}/plugins/${promote_plugin}" \ + "{\"additionalPartitions\":1,\"expectedVersion\":${version}}" + if [[ "${HTTP_CODE}" == "200" ]]; then + new_version="$(pp_json_field "${HTTP_BODY}" version)" + if [[ "${new_version}" -gt "${version}" ]]; then + pp_record_pass "scale -> v${new_version}" + version="${new_version}" + else + pp_record_fail "scale did not bump version" + fi + else + pp_record_fail "scale expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" + fi + + sleep 5 + pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" + if [[ "${HTTP_CODE}" == "200" && "$(pp_json_field "${HTTP_BODY}" version)" == "${version}" ]]; then + pp_record_pass "GET stable at v${version}" + else + pp_record_fail "GET version mismatch after mutations" + fi + + if [[ "${AUDIT_SMOKE}" == "true" ]]; then + echo " Audit pipeline smoke..." + if ./scripts/audit/verify-audit-tier3-e2e.sh --timeout "${TIMEOUT}"; then + pp_record_pass "verify-audit-tier3-e2e after mutations" + else + pp_record_fail "verify-audit-tier3-e2e failed" + fi + fi +} + +echo "Partition plan E2E (static=${RUN_STATIC}, dynamic=${RUN_DYNAMIC})" +pp_preflight_tier3 "${TIMEOUT}" + +if [[ "${RUN_STATIC}" == "true" && "${RUN_DYNAMIC}" == "true" && "${DO_ENABLE}" == "true" ]]; then + run_static_tests + run_dynamic_tests +elif [[ "${RUN_STATIC}" == "true" ]]; then + run_static_tests +elif [[ "${RUN_DYNAMIC}" == "true" ]]; then + run_dynamic_tests +fi + +if [[ "${RESTORE_STATIC}" == "true" ]]; then + echo "" + echo "Restoring static mode..." + pp_set_dynamic_enabled "${CONTAINER}" "false" || true +fi + +pp_print_results From d1f1bffe30f622d60fa9b1e5a4b34ed39825373f Mon Sep 17 00:00:00 2001 From: ramk Date: Wed, 24 Jun 2026 12:15:55 +0530 Subject: [PATCH 07/10] RANGER-5655: Drop audit READMEs and Docker scripts from PR scope Keep REST simplification to Java sources and unit tests only. Co-authored-by: Cursor --- ...README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md | 730 -------- ...ADME-KAFKA-PARTITION-PLAN-REGISTRY-REST.md | 1512 ----------------- .../audit/dynamic-auth-to-local-e2e-lib.sh | 299 ---- .../audit/dynamic-partition-plugin-e2e-lib.sh | 346 ---- .../scripts/audit/partition-plan-e2e-lib.sh | 624 ------- .../verify-dynamic-partition-plugin-e2e.sh | 138 -- .../verify-hdfs-dynamic-partition-e2e.sh | 189 --- .../audit/verify-partition-plan-e2e.sh | 282 --- 8 files changed, 4120 deletions(-) delete mode 100644 audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md delete mode 100644 audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md delete mode 100755 dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh delete mode 100755 dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh delete mode 100755 dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh delete mode 100755 dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh delete mode 100755 dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh delete mode 100755 dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh diff --git a/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md b/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md deleted file mode 100644 index 78e0c08139f..00000000000 --- a/audit-server/README-KAFKA-DYNAMIC-PARTITIONING-GUIDE.md +++ /dev/null @@ -1,730 +0,0 @@ - - -# Dynamic ingestor registry for Ranger audit-ingestor — guide for everyone - -This guide explains **why** and **how** Ranger moves from **static** ingestor configuration (XML at startup) to a **dynamic unified registry** at runtime — **without restarting** audit-ingestor. - -It covers both: - -1. **Kafka partition routing** — which plugin uses which `ranger_audits` partitions -2. **Service allowlist** — which principals may `POST /api/audit/access` for each Ranger repo - -Both live in one Kafka document on **`ranger_audit_partition_plan`** (`plugins`, `buffer`, and `services`). One REST API: **`/api/audit/partition-plan`**. - -It is written for operators, architects, and reviewers who need a shared mental model — **without reading the codebase**. - -**Confluence:** [Dynamic Ingestor Registry Guide (Ranger Audit Ingestor)](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/12043681813) (child of [Ranger Engineering](https://cloudera.atlassian.net/wiki/spaces/ENG/pages/759726545/Ranger+Engineering)) - -**Related design docs:** [README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md](README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md) · [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) - -## 1. What problem are we solving? - -Ranger plugins (HDFS, Hive, Trino, Ozone OM, etc.) send audit events to **audit-ingestor**, which writes them to Kafka (`ranger_audits`). - -Ingestor performs **two** configuration jobs: - -| Job | Question | Static today | Dynamic goal | -|-----|----------|--------------|--------------| -| **Service allowlist** | May this Kerberos principal POST audits claiming repo `R`? | XML `service..allowed.users` at startup | `services` map in unified registry | -| **Partition routing** | After accept, which Kafka partition? | XML `configured.plugins` at startup | `plugins` / `buffer` in unified registry | - -**Today (static mode):** - -- Allowlist and partition layout are defined in **XML** at startup. -- Adding a repo or plugin usually means: edit XML → **restart ingestor** on every pod. - -**Goal (dynamic mode):** - -- Change allowlist **and** partition assignments **while ingestor is running**. -- Onboard a new plugin/repo without reshuffling existing partition assignments (append-only). -- Keep all ingestor replicas consistent via one shared Kafka compacted topic. - ---- - -## 2. Core ideas (plain language) - -### Kafka topic partitions - -Think of a Kafka topic as a queue split into numbered lanes: partition `0`, `1`, `2`, … - -- Each audit event is routed to **one** partition based on the plugin id (Kafka record **key**). -- More partitions → more parallel consumption downstream (Solr/HDFS dispatchers). - -Kafka allows **increasing** partition count; it does **not** support shrinking. - -### Plugin id - -The plugin id (also called agent id / app id) identifies the source of the audit — e.g. `hdfs`, `hiveServer2`, `trino`. - -### Partition plan (unified registry document) - -A **partition plan** is a versioned JSON document stored in `ranger_audit_partition_plan`. It answers: - -- For each known plugin: **which partition numbers** may receive its audits? (`plugins` + `buffer`) -- For each Ranger service repo: **which short usernames** may `POST /access`? (`services`) -- What is the current **`topicPartitionCount`** (must match Kafka)? - -Example (simplified): - -```json -{ - "topic": "ranger_audits", - "version": 12, - "topicPartitionCount": 48, - "plugins": { - "hdfs": { "partitions": [0, 1, 2, 3, 4, 5] }, - "hiveServer2": { "partitions": [6, 7, 8, 9, 10, 11] } - }, - "buffer": { "partitions": [12, 13, 14, "... through 47 ..."] }, - "services": { - "dev_hive": { "allowedUsers": ["hive"] }, - "dev_ozone": { "allowedUsers": ["om", "ozone"] }, - "dev_trino": { "allowedUsers": ["trino"] } - } -} -``` - -The `version` field increments on every successful admin change (optimistic locking) — **one version** for routing and allowlist mutations. - -Every ingestor pod uses the **same** document so routing and authorization stay consistent. - -### Service allowlist (`POST /api/audit/access`) - -Plugins authenticate (Kerberos/JWT), then ingestor checks `services[serviceName].allowedUsers` (short names after `auth_to_local`). Failure → **403** before any Kafka produce. This check is **orthogonal** to partition routing but stored in the **same** registry document when dynamic mode is on. - -### Unified registry (source of truth) - -The live plan lives in **durable shared storage** that all ingestor replicas read — a Kafka **compacted** topic (`ranger_audit_partition_plan`). - -- No new database or ZooKeeper dependency. -- Survives pod restarts. -- All replicas see the same latest plan. - -### Append-only growth - -When a plugin needs more capacity: - -1. Increase the audit topic’s partition count (add lanes at the **tail**). -2. Assign **only the new** partition numbers to that plugin. -3. **Do not** move partitions away from other plugins. - -### Buffer partitions - -Plugins not yet in the plan (or newly appearing in the fleet) go to **buffer** partitions until an operator **promotes** them to dedicated partitions. - ---- - -## 3. Today vs proposed (at a glance) - -| | Static (today) | Dynamic (unified registry) | -|---|----------------|---------------------------| -| **Where config lives** | XML on each pod at startup | `ranger_audit_partition_plan` (`plugins` + `services`) | -| **Change allowlist or routing** | Edit XML + restart | `/api/audit/partition-plan` REST; no restart | -| **Add new plugin/repo** | XML + restart | `POST .../plugins` (onboard with mandatory `services`) | -| **Scale hot plugin** | Edit overrides + restart | `PATCH .../plugins/{pluginId}` (append-only tail partitions) | -| **Multi-replica ingestor** | Same XML if synced via ConfigMap | All pods watch same Kafka document | -| **Feature flag** | Default behavior | `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=true` | - ---- - -## 4. Static → dynamic cutover — direct answers - -**Goal:** Turn on dynamic mode so each plugin keeps sending audits to the **same Kafka partitions** it uses today — no surprise rerouting on cutover day. - -### When dynamic mode is off - -| Question | Answer | -|----------|--------| -| Is `ranger_audit_partition_plan` created? | **No** — the plan topic is not created or used. | -| How is routing decided? | From XML at startup (static mode), same as today. | -| Is there a background plan sync? | **No**. | - -### When dynamic mode is on - -| Question | Answer | -|----------|--------| -| Is the plan topic created? | **Yes** — on first startup that needs the registry. | -| Where does every ingestor get the plan? | From `ranger_audit_partition_plan` (compacted topic), kept in memory on each pod. | -| Can XML edits change live routing? | **No** (once a plan message exists in Kafka). Runtime changes go through REST. | - -### How “same partitions per plugin” is achieved on cutover - -Ingestor does **not** read Kafka to discover which plugin owns which partition. Kafka stores audit **records**, not plugin ownership. - -Preservation works because the **first published plan** uses the **same layout rules as static mode**: - -| Piece | Static today | Dynamic (first plan from XML) | -|-------|--------------|-------------------------------| -| Plugin order | `configured.plugins` list | Same list | -| Partitions per plugin | Default + per-plugin overrides | Same | -| Layout | Contiguous ranges (hdfs → 0–2, next → 3–5, … buffer tail) | Same ranges as explicit partition ID lists | -| Within-plugin pick | Round-robin per plugin id | Same round-robin | -| Unknown plugin | Hash into buffer | Same buffer pool | - -**What Kafka is used for at bootstrap:** - -| Step | Reads from | Purpose | -|------|------------|---------| -| Build first plan (empty registry) | **XML only** | Plugin list, overrides, buffer → partition lists | -| Install plan (every pod) | **Kafka AdminClient** on `ranger_audits` | **Total** partition count only — must match `topicPartitionCount` in plan | -| Route each audit | **In-memory plan** | No per-event Kafka read | - -**Brownfield (existing production cluster):** Auto-bootstrap from XML is safe only when **XML, ingestor startup logs, and `kafka-topics --describe ranger_audits` all agree**. If they differ, **pre-load the plan into Kafka before enabling dynamic** (operator publishes JSON to the plan topic while dynamic is still off). - -### Plan already in Kafka vs empty registry - -```mermaid -sequenceDiagram - participant Pod as Ingestor pod - participant Plan as ranger_audit_partition_plan - participant Audits as ranger_audits - participant XML as site.xml - - Pod->>Plan: create plan topic if missing (Race A) - Pod->>Plan: read plan for key ranger_audits - alt plan message exists - Plan-->>Pod: use stored plan — skip XML bootstrap - else registry empty - Pod->>XML: build first plan from XML (plugins + services) - Pod->>Plan: re-read (Race B — peer may have published) - Pod->>Plan: publish first plan if still empty - Pod->>Plan: mandatory read-back - end - Pod->>Audits: describe topic — partition count - Pod->>Pod: validate count matches plan → load into memory -``` - -| Situation | What each pod does | -|-----------|-------------------| -| Plan **message** already in `ranger_audit_partition_plan` | Read and use it; **do not** publish a new plan from XML | -| Plan topic exists but **no message** yet | One pod publishes the first plan; others re-read and adopt the same plan (**Race B**) | -| Several pods create the plan **topic** at once | Idempotent create — **Race A**; “already exists” is success | - -**Bootstrap logic (summary):** read plan → if missing, build from XML → re-read → publish if still missing → mandatory read-back → validate partition count → install in memory. - -After the first plan is stored in Kafka, **Kafka is the source of truth** for routing. - -### Multi-pod race — is it implemented? - -| Race | Scenario | Resolution | -|------|----------|------------| -| **A** | Several pods create `ranger_audit_partition_plan` topic together | Idempotent topic create | -| **B** | Several pods publish the **first** plan when registry is empty | Re-read before and after publish; all pods install the same plan from Kafka | - -**Your rule:** *If the plan topic exists but no plan message → add the plan; otherwise use the plan from the topic* — **Yes, implemented** via the bootstrap flow above. - ---- - -## 5. How dynamic mode works (end-to-end) - -| Plane | Kafka topic | Traffic | Who reads/writes | -|-------|-------------|---------|------------------| -| **Data** | `ranger_audits` | High — every audit event | Plugins → ingestor → dispatchers | -| **Control (unified registry)** | `ranger_audit_partition_plan` (compacted) | Low — rare routing + allowlist changes | `/api/audit/partition-plan` REST + `PartitionPlanWatcher` | - -The registry is **configuration**, not audit data. Ingestor keeps the current document **in memory**; only the watcher and REST handlers touch the plan topic. - -### Architecture (control plane vs data plane) - -```mermaid -flowchart TB - subgraph plugins [Ranger plugins] - HS2[HiveServer2] - OM[Ozone OM] - end - - subgraph ops [Ops or automation] - Admin[Admin REST client] - end - - subgraph ingestor [Each audit-ingestor pod] - Access["POST /api/audit/access"] - REST["GET/PATCH/POST /partition-plan"] - Svc[PartitionPlanService] - Watcher[PartitionPlanWatcher] - Mem[(In-memory PartitionPlan
plugins + services)] - Part[Partition router] - Queue[Audit queue] - - REST --> Svc - Svc --> PlanTopic - Watcher -->|poll / consume| PlanTopic - Watcher -->|atomic swap| Mem - HS2 -->|serviceName=dev_hive| Access - OM -->|serviceName=dev_ozone| Access - Access -->|isAllowedServiceUser from services| Mem - Access -->|on 200/202| Queue - Mem --> Part - Queue --> Part - Part -->|key=plugin id| AuditTopic - end - - subgraph kafka [Kafka] - PlanTopic[(ranger_audit_partition_plan
plugins + services)] - AuditTopic[(ranger_audits)] - end - - subgraph consumers [Downstream — unchanged] - Solr[Solr dispatcher] - HDFSdisp[HDFS dispatcher] - end - - Admin -->|via load balancer| REST - Svc -->|createPartitions if needed| AuditTopic - AuditTopic --> Solr - AuditTopic --> HDFSdisp -``` - -### First startup — seeding the plan (once per cluster) - -When dynamic mode starts and the plan registry is **empty**: - -1. Ingestor enables dynamic mode. -2. Watcher finds no document in `ranger_audit_partition_plan`. -3. Ingestor reads XML (`configured.plugins` if set, else buffer-only bootstrap from `kafka.topic.partitions`; overrides, buffer, and `service.*.allowed.users`). -4. Ingestor builds and publishes the **first document** (`plugins` + `services`) to the compacted topic. -5. Ingestor loads that document into memory and begins enforcing allowlist + routing. - -**After that:** additional pods and restarts **read Kafka only** — they do not re-build from XML. - -### Every audit — the hot path - -The plan topic is **not** read on this path. - -```text -Plugin POST /api/audit/access?serviceName=dev_hive&appId=hiveServer2 - │ - ├─ 401 Authentication failed - ├─ 403 services[dev_hive] does not include short username → STOP - └─ 200/202 Allowlist passed → partition router → ranger_audits -``` - -1. Plugin POSTs audit to ingestor (authenticate). -2. `isAllowedServiceUser` reads **in-memory** `services` map (or static XML when dynamic off). -3. Partition router reads **in-memory** `plugins` / `buffer`. -4. Known plugin → round-robin within its partition list; unknown → buffer. -5. Record written to `ranger_audits`. - -### Changing the plan — admin or automation - -**On the pod that receives the REST call:** - -1. Read current plan from Kafka. -2. Validate append-only rules. -3. Grow `ranger_audits` tail if needed (Kafka AdminClient). -4. Write new plan **version** to compacted topic. -5. Return **200 OK** or **409 Conflict** (stale `expectedVersion`). - -**On every ingestor pod (~30s sync interval):** - -1. Background sync picks up new plan version. -2. Validates against live `ranger_audits` partition count. -3. Swaps plan in memory — **no restart**. - -```mermaid -sequenceDiagram - participant Admin as Admin / automation - participant REST as Ingestor REST (one pod) - participant Plan as ranger_audit_partition_plan - participant Audit as ranger_audits - participant W as Background sync (each pod) - participant Mem as In-memory plan - participant Part as Partition router - - Admin->>REST: POST /plugins or PATCH /plugins/{id} - REST->>Plan: read current plan version - REST->>Audit: createPartitions (if needed) - REST->>Plan: write new plan version - REST-->>Admin: 200 OK - - loop Every ingestor pod - W->>Plan: poll latest plan - W->>Mem: atomic swap - Note over Part,Mem: Hot path reads memory only - Part->>Audit: produce audit (key = plugin id) - end -``` - -### Rules to remember - -- **Two topics, two jobs** — plan topic = unified config (`plugins` + `services`); audit topic = data. -- **Memory on the hot path** — no per-audit read of the plan topic. -- **Kafka is the source of truth** after the first document is published. -- **Append-only growth** — new partitions only at the tail of `ranger_audits`. -- **All pods must agree** — every ingestor syncs from the same compacted topic. -- **Allowlist and routing are separate checks** — 403 on `/access` is allowlist; wrong partition is routing. - ---- - -## 6. Admin REST API (control plane) - -When dynamic mode is on, operators change routing **and** allowlists through **`/api/audit/partition-plan`** on **any** pod (usually via load balancer). Mutations are written to `ranger_audit_partition_plan`; every pod picks up changes through `PartitionPlanWatcher` (~30s). - -**Auth:** Kerberos or JWT. When `kafka.partition.plan.allowed.users` is set, only those short names may call partition-plan REST (plugin users must not). Dynamic mode off → all partition-plan calls return **503**. - -### Endpoints (three only) - -| Method | Path | Purpose | -|--------|------|---------| -| `GET` | `/api/audit/partition-plan` | Read current plan (`plugins`, `buffer`, `services`, `version`) | -| `POST` | `/api/audit/partition-plan/plugins` | **Onboard** plugin: dedicated partitions + service allowlists (one version bump) | -| `PATCH` | `/api/audit/partition-plan/plugins/{pluginId}` | **Update** onboarded plugin: scale and/or service allowlist mutations | - -Base URL example: `https://:7081/api/audit/partition-plan` - -All mutations require **`expectedVersion`** from the last `GET`. Stale version → **409 Conflict** + current plan in body. - -**Removed (consolidated above):** `PATCH /api/audit/partition-plan`, `POST /api/audit/partition-plan/services`, separate promote-only / scale-only flows. - -> **Future work (not implemented):** bootstrap **v0** split — seeding `services` from XML separately from plugin partition assignments. Current bootstrap still publishes a single v1 plan from XML. - -### Common operations - -**Read plan** - -```http -GET /api/audit/partition-plan -→ 200 + JSON plan (note the "version" field) -``` - -**Onboard plugin** — dedicated partitions + allowlists in one call (`services` is **required** and must be non-empty): - -```json -POST /api/audit/partition-plan/plugins -{ - "pluginId": "hiveServer2", - "partitionCount": 3, - "expectedVersion": 1, - "services": { - "dev_hive": { "allowedUsers": ["hive"] }, - "dev_hive2": { "allowedUsers": ["hive2"] } - } -} -``` - -Each service entry is stored with `pluginId` for ownership tracking. - -**Update plugin** — scale and/or mutate allowlists scoped to `{pluginId}` (at least one delta required): - -```json -PATCH /api/audit/partition-plan/plugins/hiveServer2 -{ - "expectedVersion": 2, - "addServices": { "dev_hive3": { "allowedUsers": ["hive3"] } }, - "removeServices": ["dev_hive2"], - "additionalPartitions": 2 -} -``` - -| Field | Purpose | -|-------|---------| -| `additionalPartitions` | int ≥ 1 — append tail partition IDs (append-only) | -| `addServices` | map repo → `{ "allowedUsers": [...] }` | -| `updateServices` | map repo → `{ "allowedUsers": [...] }` — replace allowlist for repos owned by `{pluginId}` | -| `removeServices` | list of repo names to remove (scoped to `{pluginId}`) | - -Success → **200** + updated plan JSON (version incremented on change). -Repeating the same onboard or update delta (with matching `expectedVersion`) → **200** + current plan JSON with **no** registry write or version bump. -State conflict (resource exists but request differs, e.g. different partition count) → **400**. -Stale version → **409** + current plan in body. - -### What happens inside one REST call - -```mermaid -flowchart LR - Admin[Admin or script] --> REST[Ingestor REST] - REST --> Read[Read plan from Kafka] - Read --> Check{expectedVersion OK?} - Check -->|No| R409[409 + current plan] - Check -->|Yes| Valid[Validate append-only change] - Valid --> Grow{Need more audit partitions?} - Grow -->|Yes| Topic[Grow ranger_audits tail] - Grow -->|No| Write[Write new plan to Kafka] - Topic --> Write - Write --> OK[200 + new plan] - OK --> Sync[All pods sync within ~30s] -``` - -| Step | What the ingestor does | -|------|------------------------| -| 1 | Authenticate caller; check `kafka.partition.plan.allowed.users` when configured | -| 2 | Read current plan from `ranger_audit_partition_plan` | -| 3 | Reject if `expectedVersion` does not match | -| 4 | Compute new plugin lists (append-only — no reshuffling existing slots) | -| 5 | If new partition IDs are needed → grow `ranger_audits` **first** | -| 6 | Publish new plan version to Kafka | -| 7 | Return updated plan JSON | - -**GET** is cheap (memory). **POST / PATCH** always goes through Kafka so all pods converge on the same plan. - ---- - -## 7. Operator workflow: onboarding a plugin or repo - -**Recommended:** one `POST /api/audit/partition-plan/plugins` with a **non-empty `services` map** — allowlist and dedicated partitions in one plan version. - -### Stage 0 — Create service in Ranger Admin - -1. Create service `dev_trino` in Policy Manager; set `policy.download.auth.users`. -2. Configure plugin audit destination → ingestor URL (`:7081`). - -### Stage 1 — Onboard via unified registry - -```http -POST /api/audit/partition-plan/plugins -``` - -Include `pluginId`, `partitionCount`, `expectedVersion`, and at least one repo in `services` (see [§6](#6-admin-rest-api-control-plane)). - -All ingestors apply within ~30s. **No restart** required. - -### Stage 2 — Verify plugin POST - -Expect **200/202** on `/access`, not **403**. - -### Stage 3 — Scale or change allowlists (optional) - -Call `PATCH /api/audit/partition-plan/plugins/{pluginId}` to add tail partitions, add/update/remove repos, or combine all in one version bump. - -**Do not** edit `ranger-audit-ingestor-site.xml` on one pod for runtime changes. XML is only for **initial bootstrap** when the registry is empty. - ---- - -## 8. Operator workflow: partition-only changes - -Use when an onboarded plugin already has allowlists and you only need routing changes (scale tail partitions). - -### Stage 0 — Plugin appears (unknown) - -- Audits use **buffer** partitions. -- Monitor volume per plugin id. - -### Stage 1 — Onboard plugin (partitions + allowlists) - -- Call `POST /api/audit/partition-plan/plugins` with mandatory `services` (see [§6](#6-admin-rest-api-control-plane)). -- All ingestors apply within ~30s. **No restart** required. - -### Stage 2 — Scale a hot plugin - -- Call `PATCH /api/audit/partition-plan/plugins/{pluginId}` with `additionalPartitions`. -- Dispatchers rebalance automatically when the audit topic grows. - -**Do not** edit `ranger-audit-ingestor-site.xml` on one pod for runtime changes. XML is only for **initial bootstrap** when the plan registry is empty. - ---- - -## 9. Configuration (dynamic mode) - -| Property | Purpose | Example | -|----------|---------|---------| -| `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` | Turn dynamic unified registry on/off | `false` (default) = static XML | -| `ranger.audit.ingestor.kafka.partition.plan.topic` | Compacted registry topic name | `ranger_audit_partition_plan` | -| `ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms` | How often pods reload plan | `30000` | -| `ranger.audit.ingestor.kafka.partition.plan.allowed.users` | Who may call partition-plan REST | `admin,ops` | -| `ranger.audit.ingestor.service..allowed.users` | Static bootstrap per-repo allowlist | `hive`, `om,ozone`, … | -| `ranger.audit.ingestor.auth.to.local` | Principal → short name rules | Same as Hadoop `hadoop.security.auth_to_local` | - -When dynamic is **off**, routing and allowlist are fixed from XML at startup; the plan topic is not used. - -When dynamic is **on** and the registry is **empty**, the first ingestor pod seeds the plan (`plugins` + `services`) from XML. Later pods and restarts read **Kafka only**. - ---- - -## 10. FAQ - -### Basics - -**Why plugin-based partitioning?** -Hot plugins (HDFS, Hive) can get dedicated Kafka lanes so they do not starve others. Unknown plugins share a buffer until you promote them. - -**What is the difference between static and dynamic mode?** -Static: routing is computed once from XML at startup; changes need restart. Dynamic: routing lives in Kafka; admins change it via REST while ingestor keeps running. - -**Why not store the plan in Postgres or ZooKeeper?** -Ingestor already requires Kafka. A compacted plan topic adds no new infrastructure. - -**Why not edit XML on a running pod?** -Each pod has its own copy; edits are not shared, not durable, and are lost on restart. Runtime changes belong in the plan topic via REST. - -**Can I change routing by editing XML while dynamic mode is on?** -No for live routing. Ingestor uses the Kafka plan in memory, not XML edits on disk. Update XML only when preparing static rollback or documenting the intended layout. - -**Can I decrease `ranger_audits` partition count?** -No. Kafka does not support shrinking partitions. You can only add partitions at the tail. - -### Two topics and sync - -**What are the two Kafka topics?** -`ranger_audits` = audit data (high volume). `ranger_audit_partition_plan` = routing config (low volume, compacted). - -**Does every audit POST read the plan topic?** -No. Each audit uses the plan already in **memory** on that pod. Only background sync and REST mutations touch the plan topic. - -**How do all ingestor pods stay in sync?** -Every pod watches the same compacted plan topic (default every 30s) and swaps the new plan into memory. - -**What happens when a pod restarts?** -It reads the latest plan from Kafka (if dynamic is on). The plan survives in Kafka across crashes. - -**Do Solr and HDFS dispatchers need the partition plan?** -No. They consume **all** partitions of `ranger_audits`. Only ingestor uses the plan to **choose** which partition to write to. - -**Will changing the plan break consumers?** -Adding partitions triggers normal consumer rebalance. Existing plugin slots keep the same partition numbers if you follow append-only promote/scale rules. - -### Plan content and routing - -**What is the buffer?** -Partitions reserved for plugins that are not yet promoted (or newly seen plugin ids). Promote moves a plugin from buffer to dedicated slots. - -**What does append-only mean?** -When scaling, only **new** partition numbers at the end of `ranger_audits` are assigned. Existing plugins keep the same partition IDs in the same order. - -**How is a partition chosen inside a plugin’s list?** -Round-robin per plugin id — same behavior as static mode. - -**Where does an unknown plugin send audits?** -To the buffer partition pool (hash-based pick within buffer list in dynamic mode). - -### REST and concurrency - -**Do I need to restart ingestor after promote or scale?** -No. Background sync applies the new plan within about one refresh interval (~30s). - -**What is `expectedVersion`?** -The plan `version` you believe is current when you write. If someone else published first, your version is stale and you get **409**. - -**What should I do on HTTP 409?** -Another writer published a newer plan. Use the plan in the 409 response (or `GET` again), note the new `version`, and retry your change with that `expectedVersion`. - -**Why grow `ranger_audits` before publishing a new plan?** -The plan must not reference partition IDs that do not exist yet. The server grows the audit topic tail first, then writes the plan. - -**Why does promote return 400?** -Common cases: plugin already has dedicated partitions, invalid partition count, or plugin id missing. Scale returns 400 if the plugin is not in the plan yet (promote it first). - -**What do the HTTP status codes mean?** -**503** — dynamic mode is off, or the server could not grow `ranger_audits` (Kafka admin failure). **400** — validation failed (bad shape, append-only violation, `topicPartitionCount` ≠ Kafka). **409** — version conflict; retry with the plan body returned in the response. - -**Do dispatchers need reconfiguration after scale?** -No. Consumer groups rebalance automatically when partition count increases. Tune consumer threads only if you see sustained lag. - -### Cutover and bootstrap - -**When is `ranger_audit_partition_plan` created?** -Only when dynamic mode is enabled. With dynamic off, the topic is not created or used. - -**Does bootstrap read Kafka to learn plugin → partition mapping?** -No. The first plan is built from XML (same layout as static mode). Kafka is used for **total** partition count validation and as the durable registry. - -**Can I publish the plan to Kafka before enabling dynamic mode?** -Yes — recommended for brownfield clusters. Ingestor will read your pre-loaded plan and will not replace it with a fresh XML bootstrap. - -**Greenfield vs brownfield cutover — what is different?** -Greenfield: enable dynamic on an empty registry; first pod seeds from XML (empty `configured.plugins` → buffer-only plan sized by `kafka.topic.partitions`). Brownfield: export static layout, pre-seed the plan topic (or set `configured.plugins` in XML to match production), then enable dynamic and verify every pod shows the same plan. - -**Does the `configured.plugins` XML list still matter in dynamic mode?** -Yes for **bootstrap only** when the registry is empty. After the first plan exists, runtime routing changes go through REST, not by editing that list. - -**How do I verify cutover succeeded?** -`GET /api/audit/partition-plan` on every pod (same `version`); plugin lists match your saved static logs; `topicPartitionCount` matches `kafka-topics --describe ranger_audits`. - -**How do I roll back to static mode?** -Export current plan via `GET`, align XML to that layout if needed, set `dynamic.enabled=false`, rolling restart. Plan topic remains but is ignored. - -**What if multiple pods start together with an empty registry?** -Race A: idempotent plan-topic create. Race B: re-read before/after first publish; all pods install the same plan. See [§4](#4-static--dynamic-cutover--direct-answers). - -### Troubleshooting - -**Ingestor fails startup after enabling dynamic — what now?** -Often Kafka unreachable or cannot create/read the plan topic. Fix Kafka connectivity; keep dynamic off until Kafka is healthy. With Kafka down at startup, ingestor should **not** come up healthy in dynamic mode. - -**Pods show different plan versions — what now?** -Wait one refresh interval after a change. If still mismatched, check plan topic readability and watcher logs on the lagging pod. - -**Does audit recovery / local spool behavior change?** -No. Per-pod spool and retry when Kafka is briefly unavailable works as today. - -**Does this replace producer throughput tuning?** -No. Batch size, linger, and compression are separate settings. - -**Who can call partition-plan REST?** -Ops principals in `kafka.partition.plan.allowed.users` when configured (Kerberos/JWT). Plugin users (`hive`, `om`) send audits via `/access` — they must **not** mutate the registry. - -### Service allowlist (`POST /api/audit/access`) - -**Why do we need an allowlist if Kerberos already authenticates the plugin?** -Authentication proves *who* connected. Authorization proves they may **claim audits for this repo**. Kerberos success alone would let any daemon POST as any `serviceName`. - -**Does dynamic partition plan remove the allowlist check?** -**No.** Partition routing runs **after** `/access` accepts the batch. **403** on `/access` is always a **service allowlist** failure (not routing). - -**What is `serviceName`?** -The Ranger Policy Manager **repo name** (e.g. `dev_hive`), not the service type (`hive`). - -**What goes in `allowedUsers`?** -Short names after `auth_to_local` — same values as `policy.download.auth.users` on the Ranger service (or a subset). - -**Consistency rule (ops discipline):** -`allowedUsers for repo R ⊆ { short names from policy.download.auth.users for R in Ranger Admin }`. Ranger Admin does not auto-sync in Phase 1; ingestor may reject REST writes that violate subset rules (strict mode). - -**Three layers — do not merge:** - -| Layer | Who | Purpose | -|-------|-----|---------| -| **Service allowlist** (plugin POST) | Daemons (`hive`, `om`, …) | May this principal POST audits for repo `R`? | -| **Partition plan** (Kafka routing) | Ingestor internal | Which Kafka partition after accept? | -| **Admin REST APIs** | Ops (`admin`, `ops`, …) | Who may change the unified registry via `/api/audit/partition-plan`? | - -**If you see…** - -| Symptom | Layer | Fix | -|---------|-------|-----| -| **401** on `/access` | Authentication | Plugin / ingestor Kerberos, keytabs, SPNEGO | -| **403** on `/access` | **Service allowlist** | Add repo via `POST .../plugins` (onboard) or `PATCH .../plugins/{pluginId}` (`addServices` / `updateServices`) | -| Audits accepted but wrong Kafka partition | **Partition plan** | `POST .../plugins` (onboard) / `PATCH .../plugins/{pluginId}` (scale) | -| New repo in Admin, audits still **403** | Allowlist not onboarded | `POST .../plugins` or `PATCH .../plugins/{pluginId}` with `addServices` | -| Allowlist OK, audits in buffer partition | Partition plan not onboarded | `POST .../plugins` with `partitionCount` + `services` | - -**Plugin gets 403 but Kerberos works — what now?** -Short name not in `services[repo].allowedUsers`, wrong `serviceName`, or `auth_to_local` mismatch. - -**New repo in Admin UI, still 403?** -Creating the service in Admin does not auto-update ingestor (Phase 1). Call `POST /api/audit/partition-plan/plugins` (new plugin) or `PATCH .../plugins/{pluginId}` with `addServices`. - -**Do I need both allowlist and partition plan for a new plugin?** -Yes for full production onboarding. `POST /api/audit/partition-plan/plugins` requires a non-empty `services` map and assigns dedicated partitions in **one** plan version. - -**Is there a bundled API?** -`POST /api/audit/partition-plan/plugins` registers `services[repo]` allowlists and promotes plugin partitions atomically. - ---- - -## 11. Docs - -| Doc | Purpose | -|-----|---------| -| [README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md](README-DYNAMIC-SERVICE-ALLOWLIST-DESIGN.md) | Service allowlist design (unified `services` map) | -| [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) | Partition plan architecture | -| [README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md](README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md) | Partition-plan Kafka topic + REST | -| [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) | Partition-plan implementation phases | -| [DESIGN-KAFKA-AUDIT-SERVER.md](DESIGN-KAFKA-AUDIT-SERVER.md) | End-to-end audit pipeline | -| [PR #1017](https://github.com/apache/ranger/pull/1017) (RANGER-5645) | Static Docker allowlist fix | diff --git a/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md b/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md deleted file mode 100644 index aac724dbfa6..00000000000 --- a/audit-server/README-KAFKA-PARTITION-PLAN-REGISTRY-REST.md +++ /dev/null @@ -1,1512 +0,0 @@ - - -# Kafka-backed partition plan registry + REST (detailed design) - -> **Consolidated design (start here for review):** [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) — architecture, flows, diagrams, brownfield migration, and Q&A in one document. This README is the deep dive on Kafka registry + REST semantics. - -This document describes the **recommended** approach for **dynamic plugin onboarding** and **dynamic partition scaling** in `audit-ingestor` without Postgres or ZooKeeper. - -**Related docs:** - -- **Consolidated design:** [DESIGN-KAFKA-DYNAMIC-PARTITIONING.md](DESIGN-KAFKA-DYNAMIC-PARTITIONING.md) -- Current static partitioning: `README-KAFKA-PLUGIN-PARTITIONING.md` -- Design summary + checklist: `README-KAFKA-PLUGIN-PARTITIONING-DYNAMIC-DESIGN.md` -- **Phased implementation plan:** [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) -- Producer performance gaps: `README-KAFKA-PRODUCER-PERFORMANCE.md` -- Solr/HDFS consumers: `README-KAFKA-DISPATCHERS.md` - ---- - -## Problem we are solving - -Today (`README-KAFKA-PLUGIN-PARTITIONING.md`): - -- Plugin → partition mapping is computed once at ingestor startup from XML. -- Changing `configured.plugins` or overrides requires restart. -- Contiguous-range allocation **reshuffles** later plugins when an early plugin’s count changes. - -Goal: - -- **Onboard** plugins (e.g. trino) and **scale** hot plugins (e.g. hiveServer2) **without restart**. -- **No new infra** (no Postgres/ZooKeeper). -- **All ingestor replicas** use the same routing after a change. -- **Survive pod crash/restart** without losing the plan. - ---- - -## High-level architecture - -### In plain terms - -There are **two separate Kafka topics** and **two different jobs**: - -| Topic | What it stores | Who cares | -|-------|----------------|-----------| -| **`ranger_audit_partition_plan`** | Small JSON: which plugin uses which partition IDs | **Ingestor only** (config) | -| **`ranger_audits`** | Actual audit events | **Ingestor produces**, **Solr/HDFS consume** | - -**Ingestor pods do three things:** - -1. **Serve audits** — plugins POST `/access` → pick partition from **in-memory plan** → write to `ranger_audits` (fast; no Kafka read per audit). -2. **Watch the plan** — background thread reads `ranger_audit_partition_plan` and updates memory when the plan changes. -3. **Admin REST** — ops call `GET /api/audit/partition-plan`, `POST .../plugins` (onboard), or `PATCH .../plugins/{pluginId}` (update). - -Solr and HDFS dispatchers **never read the plan**. They read **all** partitions of `ranger_audits`. - ---- - -### Picture (control plane vs data plane) - -```mermaid -flowchart TB - subgraph ops [Ops / automation] - Admin[Admin REST client] - end - - subgraph ingestor [audit-ingestor pods A, B, C ...] - REST[AuditREST partition-plan API] - Svc[PartitionPlanService] - Watcher[PartitionPlanWatcher background thread] - Mem[(In-memory PartitionPlan)] - Part[AuditPartitioner] - Queue[AuditMessageQueue] - - REST --> Svc - Svc --> PlanTopic - Watcher --> PlanTopic - Watcher --> Mem - Mem --> Part - Plugin[Plugin POST /access] --> Queue --> Part - Part --> AuditTopic - end - - subgraph kafka [Kafka] - PlanTopic[(ranger_audit_partition_plan compacted)] - AuditTopic[(ranger_audits)] - end - - subgraph consumers [Unchanged consumers] - Solr[Solr dispatcher] - HDFS[HDFS dispatcher] - end - - Admin -->|any pod via LB| REST - Svc -->|createPartitions if needed| AuditTopic - AuditTopic --> Solr - AuditTopic --> HDFS -``` - -**Takeaway:** plan topic = **slow, rare config**; audit topic = **high volume data**. Only REST + watcher touch the plan topic. - ---- - -### The three flows (step by step) - ---- - -#### Flow 1 — Admin changes the plan - -**When:** You onboard a plugin (e.g. trino) or give a hot plugin more partitions (e.g. hiveServer2). -**How often:** Rare — ops or automation, not every audit. - -| Step | What happens | -|------|----------------| -| 1 | Admin sends REST to **any** ingestor pod (`GET` to read, `POST .../plugins` to onboard, `PATCH .../plugins/{pluginId}` to update). | -| 2 | That pod reads the current plan from **`ranger_audit_partition_plan`** (e.g. version 4). | -| 3 | If more partitions are needed, pod grows **`ranger_audits`** via Kafka AdminClient. | -| 4 | Pod writes the **new plan** (version 5) to the compacted topic. | -| 5 | Pod returns **200 OK** to admin (or **409** if another pod updated first — retry with new version). | -| 6 | **Every ingestor pod** — background watcher loads version 5 into memory (~30s or on Kafka message). | -| 7 | **Solr / HDFS** — no config change; they rebalance only if `ranger_audits` got more partitions. | - -```mermaid -flowchart TB - Admin[Admin / automation] - - subgraph step1 [Ingestor pod handles REST] - REST[Ingestor REST] - PlanRW[(Plan topic
read + write)] - Grow[(ranger_audits
grow if needed)] - REST --> PlanRW - REST --> Grow - end - - subgraph step2 [All ingestor pods sync] - Watcher[Watcher on each pod] - Mem[Memory updated] - Same[Same routing everywhere] - Watcher --> Mem --> Same - end - - subgraph step3 [Dispatchers unchanged] - SolrHDFS[Solr / HDFS] - Consume[Keep consuming ranger_audits] - SolrHDFS --> Consume - end - - Admin --> REST - PlanRW --> Watcher - Grow -.->|rebalance only if partitions grew| SolrHDFS -``` - ---- - -#### Flow 2 — Plugin sends an audit - -**When:** Every time a Ranger plugin reports an access event. -**How often:** High volume — this is the hot path. - -| Step | What happens | -|------|----------------| -| 1 | Plugin sends **POST `/access`** to ingestor (any pod via load balancer). | -| 2 | Ingestor reads the plan **already in memory** (not from Kafka). | -| 3 | Ingestor finds partitions for this plugin id (`agentId`), e.g. hive → `[4,5,6,7,8,9]`. | -| 4 | Ingestor picks one partition (round-robin within that list). | -| 5 | Ingestor writes the audit event to **`ranger_audits`**. | - -**Important:** This path **never** reads `ranger_audit_partition_plan`. Fast and cheap. - -```mermaid -flowchart LR - Plugin[Plugin] --> Access[POST /access] - Access --> Mem[(Memory
plan)] - Mem --> Pick[Pick partition
for agentId] - Pick --> Audit[(ranger_audits)] - - Watcher[Watcher
Flow 1 or Flow 3] -.->|fills memory| Mem -``` - ---- - -#### Flow 3 — Pod startup (first pod vs others) - -**When:** Ingestor pod starts or restarts. -**Goal:** Every pod ends up with the **same plan in memory**. - -**Prerequisite for XML → Kafka bootstrap:** `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled=true`. If `false` or absent, legacy XML-only mode applies and **`ranger_audit_partition_plan` is not populated**. - -| Situation | What the pod does | -|-----------|-------------------| -| **Pod 1** — plan topic is **empty** | Read XML → build plan **v1** → **write v1 to `ranger_audit_partition_plan`** → load into memory. | -| **Pod 2, 3, …** — plan **already in Kafka** | **Read plan from Kafka only** → load into memory. **Do not** use XML for routing. | -| **Any pod restarts** | Same as Pod 2+ — read from Kafka, not XML. | - -```mermaid -flowchart TB - subgraph pod1 [First pod ever — plan topic empty] - direction LR - XML[XML config] --> Kafka1[(ranger_audit_partition_plan v1)] - Kafka1 --> Mem1[Memory] - end - - subgraph podN [Later pods — plan already in Kafka] - direction LR - Kafka2[(ranger_audit_partition_plan)] --> Mem2[Memory] - XML2[XML ignored for routing] - XML2 -.->|not used| Mem2 - end - - subgraph podR [Pod crash or restart] - direction LR - Kafka3[(ranger_audit_partition_plan
plan survives in Kafka)] --> Mem3[Memory] - end -``` - -**Rule of thumb:** After v1 exists in Kafka, **Kafka is the source of truth** — not XML on the pod. - -**Full bootstrap details** (XML properties, example v1 JSON, when bootstrap does *not* run): [Bootstrap and multi-pod startup → First pod: XML populates the plan topic](#first-pod-xml-populates-the-plan-topic). - ---- - -### Who talks to what (quick reference) - -| Actor | Reads plan topic | Writes plan topic | Reads/writes ranger_audits | -|-------|------------------|-------------------|----------------------------| -| Plugin audit POST | No | No | Produce only | -| PartitionPlanWatcher | Yes (background) | No | No | -| REST GET/POST/PATCH | Yes | Yes (POST/PATCH mutations) | AdminClient grow only | -| Solr dispatcher | No | No | Consume all partitions | -| HDFS dispatcher | No | No | Consume all partitions | - ---- - -## Code map: design vs `audit-server` today - -The dynamic partition plan is **design + README only** — the classes below marked **Proposed** do not exist in the repo yet. This section maps the design to **current audit-ingestor code** so you can see what to extend. - -### Module layout - -| Module | Path | Role | -|--------|------|------| -| **audit-ingestor** | `audit-server/audit-ingestor/` | HTTP REST, Kafka producer, recovery | -| **audit-common** | `audit-server/audit-common/` | Shared constants, `AuditMessageQueueUtils` (AdminClient) | -| **audit-dispatcher** | `audit-server/audit-dispatcher/` | Solr/HDFS consumers (unchanged by partition plan) | - -### Design component → Java / config (exists today vs proposed) - -| Design component | Status | Location in repo | -|------------------|--------|------------------| -| **`PartitionPlan`** (JSON model) | **Proposed** | New class, e.g. `audit-ingestor/.../partition/PartitionPlan.java` | -| **`PartitionPlanAllocator`** | **Proposed** | New — append-only tail allocation | -| **`PartitionPlanRegistry`** | **Proposed** | New — Kafka compacted topic read/write | -| **`PartitionPlanService`** | **Proposed** | New — REST handler logic | -| **`PartitionPlanWatcher`** | **Proposed** | New — background thread per pod | -| **`AuditPartitioner`** | **Exists** (static XML) | `audit-ingestor/.../kafka/AuditPartitioner.java` | -| **`AuditREST`** `/access` | **Exists** | `audit-ingestor/.../rest/AuditREST.java` | -| **`AuditREST`** `/partition-plan` | **Proposed** | Same file — new `@GET` / `@PATCH` / `@POST` methods | -| **`AuditMessageQueue`** | **Exists** | `audit-ingestor/.../kafka/AuditMessageQueue.java` | -| **`AuditProducer`** | **Exists** | `audit-ingestor/.../kafka/AuditProducer.java` | -| **Topic create + `createPartitions`** | **Exists** | `audit-common/.../AuditMessageQueueUtils.java` | -| **XML partition config** | **Exists** | `audit-ingestor/.../conf/ranger-audit-ingestor-site.xml` | -| **`partition.plan.dynamic.enabled`** etc. | **Proposed** | Not in `AuditServerConstants.java` or site XML yet | - ---- - -### Flow 2 in code today (plugin POST → Kafka) - -**Description:** Every plugin audit hits the same Java call chain today. Partition key is `agentId` (plugin id). `AuditPartitioner` picks the Kafka partition from **static XML** at startup (not from plan topic yet). - -| Step | Class / file | -|------|----------------| -| 1 | `AuditREST.logAccessAudit()` — `audit-ingestor/.../rest/AuditREST.java` | -| 2 | `AuditDestinationMgr.logBatch()` — `audit-ingestor/.../producer/AuditDestinationMgr.java` | -| 3 | `AuditMessageQueue.log(batch)` — `audit-ingestor/.../kafka/AuditMessageQueue.java` | -| 4 | `AuditProducer.sendBatch()` — `audit-ingestor/.../kafka/AuditProducer.java` | -| 5 | `KafkaProducer` key=`agentId` → `AuditPartitioner.partition()` | -| 6 | Topic **`ranger_audits`** | - -```mermaid -flowchart LR - Plugin[Ranger plugin] -->|POST /audit/access| REST[AuditREST] - REST --> Mgr[AuditDestinationMgr] - Mgr --> Queue[AuditMessageQueue] - Queue -->|key = agentId| Prod[AuditProducer] - Prod --> Part[AuditPartitioner] - Part --> Topic[(ranger_audits)] -``` - -**Partition key** in `AuditMessageQueue.java` — `authzEvent.getAgentId()`. - -**Custom partitioner** wired in `AuditProducer.java` when `configured.plugins` is set → `AuditPartitioner`. - ---- - -### Static partitioning today (`AuditPartitioner` = future bootstrap v1 logic) - -At startup, `configure()` reads the same XML properties the design uses for **first-pod bootstrap**. It builds **contiguous ranges** (not explicit JSON lists yet): - -```56:102:audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java - public void configure(Map configs) { - String propPrefix = AuditServerConstants.PROP_PREFIX_AUDIT_SERVER; - - String pluginsStr = getConfig(configs, propPrefix + AuditServerConstants.PROP_CONFIGURED_PLUGINS, AuditServerConstants.DEFAULT_CONFIGURED_PLUGINS); - configuredPlugins = pluginsStr.split(","); - // ... - defaultPartitionsPerPlugin = getIntConfig(configs, propPrefix + AuditServerConstants.PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN, ...); - // ... - for (int i = 0; i < configuredPlugins.length; i++) { - String overrideKey = propPrefix + AuditServerConstants.PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX + plugin; - int partitionCount = getIntConfig(configs, overrideKey, defaultPartitionsPerPlugin); - pluginPartitionCounts[i] = partitionCount; - } - // contiguous ranges → configuredPluginPartitionStart/End, bufferPartitionStart/Count -``` - -**Runtime routing** (round-robin for configured plugin, hash for buffer): - -```117:151:audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java - public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { - // ... - int pluginIndex = indexOfConfiguredPlugin(appId); - if (pluginIndex >= 0) { - // round-robin within plugin range - return start + subPartition; - } else { - // Unconfigured plugin - use buffer partitions - int p = Math.abs(appId.hashCode() % count) + start; - return Math.min(p, numPartitions - 1); - } - } -``` - -**Dynamic design change:** replace fixed `int[]` ranges with `AtomicReference` and explicit `[0,1,2,...]` lists from Kafka. - ---- - -### Topic size + `AdminClient.createPartitions` (reuse for REST POST/PATCH) - -Topic creation and partition **increase** already exist — the proposed `PartitionPlanService` should call the same patterns: - -```49:116:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java - public static String createAuditsTopicIfNotExists(Properties props, String propPrefix) { - // ... - int partitions = getPartitions(props, propPrefix); - // AdminClient.createTopics or updateExistingTopicPartitions -``` - -```230:249:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java - if (partitions > currentPartitions) { - // ... - newPartitionsMap.put(topicName, NewPartitions.increaseTo(partitions)); - CreatePartitionsResult createPartitionsResult = admin.createPartitions(newPartitionsMap); - createPartitionsResult.all().get(); -``` - -**Partition count from XML** (same inputs as bootstrap v1 JSON): - -```330:365:audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java - private static int getPartitions(Properties prop, String propPrefix) { - String configuredPlugins = MiscUtil.getStringProperty(prop, propPrefix + "." + AuditServerConstants.PROP_CONFIGURED_PLUGINS, ...); - if (configuredPlugins == null || configuredPlugins.trim().isEmpty()) { - totalPartitions = MiscUtil.getIntProperty(prop, propPrefix + "." + AuditServerConstants.PROP_TOPIC_PARTITIONS, ...); - } else { - for (String plugin : configuredPlugins.split(",")) { - int partitionCount = MiscUtil.getIntProperty(prop, overrideKey, defaultPartitionsPerPlugin); - totalPartitions += partitionCount; - } - totalPartitions += bufferPartitions; - } - return totalPartitions; - } -``` - ---- - -### REST today vs proposed admin API - -**Exists** — plugin audit ingestion only: - -| Endpoint | File | -|----------|------| -| `GET /audit/health` | `AuditREST.java` | -| `GET /audit/status` | `AuditREST.java` | -| `POST /audit/access` | `AuditREST.java` — auth via `allowed.users` per service | - -**Proposed** — not in code yet: - -| Endpoint | Intended location | -|----------|-------------------| -| `GET /api/audit/partition-plan` | `AuditREST.java` | -| `POST /api/audit/partition-plan/plugins` | `PartitionPlanService.onboardPlugin` | -| `PATCH /api/audit/partition-plan/plugins/{pluginId}` | `PartitionPlanService.updatePlugin` | - ---- - -### Configuration constants today (`AuditServerConstants`) - -Properties used for **legacy / bootstrap** partitioning (no `partition.plan.*` yet): - -```56:85:audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java - public static final String PROP_TOPIC_PARTITIONS = "kafka.topic.partitions"; - public static final String PROP_PARTITIONER_CLASS = "kafka.partitioner.class"; - public static final String PROP_CONFIGURED_PLUGINS = "kafka.configured.plugins"; - public static final String PROP_TOPIC_PARTITIONS_PER_CONFIGURED_PLUGIN = "kafka.topic.partitions.per.configured.plugin"; - public static final String PROP_PLUGIN_PARTITION_OVERRIDE_PREFIX = "kafka.plugin.partition.overrides."; - public static final String PROP_BUFFER_PARTITIONS = "kafka.topic.partitions.buffer"; - // ... - public static final int DEFAULT_PARTITIONS_PER_CONFIGURED_PLUGIN = 3; - public static final int DEFAULT_BUFFER_PARTITIONS = 9; -``` - -**Sample XML** — `audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml`: - -- `ranger.audit.ingestor.kafka.configured.plugins` -- `ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin` -- `ranger.audit.ingestor.kafka.topic.partitions.buffer` -- `ranger.audit.ingestor.kafka.plugin.partition.overrides.*` - ---- - -### Where new code plugs in - -**Description:** Solid boxes = **exists today**. Dashed = **proposed**. Audit hot path unchanged; dynamic plan adds watcher + admin REST + plan topic. - -```mermaid -flowchart TB - subgraph exists [Exists today] - Plugin[Plugin POST /access] - RESTA[AuditREST /access] - Queue[AuditMessageQueue] - Prod[AuditProducer] - Part[AuditPartitioner static XML] - Audits[(ranger_audits)] - Plugin --> RESTA --> Queue --> Prod --> Part --> Audits - end - - subgraph proposed [Proposed] - RESTP[AuditREST /partition-plan] - Svc[PartitionPlanService] - Watcher[PartitionPlanWatcher] - Plan[(ranger_audit_partition_plan)] - Admin[Admin / automation] - Admin --> RESTP --> Svc - Svc --> Plan - Svc -->|createPartitions| Audits - Watcher -->|read| Plan - Watcher -->|update memory| Part - end -``` - ---- - -## Core components (new code) - -| Component | Responsibility | -|-----------|----------------| -| **`PartitionPlan`** | Immutable snapshot: plugin → explicit partition ID list, buffer list, `version`, `topicPartitionCount`, `updatedAt` | -| **`PartitionPlanAllocator`** | Append-only rules: assign new tail partitions; never remove/reassign existing plugin partitions | -| **`PartitionPlanRegistry`** | Read/write plan to Kafka compacted topic | -| **`PartitionPlanService`** | Orchestrates GET/POST/PATCH: validate, allocate, AdminClient, registry write | -| **`PartitionPlanWatcher`** | Background thread on **every** ingestor pod; refreshes in-memory plan | -| **`AuditPartitioner` (modified)** | Route by plan: configured plugin → round-robin within its list; unknown → buffer list | -| **`AuditREST` (new endpoints)** | Admin API; AuthZ like `/access` | - -### Existing code to reuse (already in repo) - -**Description:** Build new partition-plan components on these classes — do not rewrite topic admin or produce plumbing. - -| Reuse | Path | Use for | -|-------|------|---------| -| `AuditMessageQueueUtils.createAuditsTopicIfNotExists()` | `audit-common/.../AuditMessageQueueUtils.java` | Pattern for `createPlanTopicIfNotExists()` | -| `AuditMessageQueueUtils.updateExistingTopicPartitions()` | same | REST POST/PATCH → grow `ranger_audits` | -| `AuditProducer` | `audit-ingestor/.../kafka/AuditProducer.java` | Existing Kafka producer; plan topic produce | -| `AuditPartitioner` | `audit-ingestor/.../kafka/AuditPartitioner.java` | Add `AtomicReference` | -| `AuditREST` + `isAllowedServiceUser()` | `audit-ingestor/.../rest/AuditREST.java` | Admin AuthZ pattern for `/partition-plan` | - -```mermaid -flowchart LR - subgraph new [Proposed new code] - PP[PartitionPlan] - Reg[PartitionPlanRegistry] - Svc[PartitionPlanService] - Watch[PartitionPlanWatcher] - end - - subgraph reuse [Reuse from repo] - Utils[AuditMessageQueueUtils] - Part[AuditPartitioner] - REST[AuditREST] - Prod[AuditProducer] - end - - Svc --> Utils - Svc --> Reg - Svc --> Prod - Watch --> Reg - Watch --> Part - REST --> Svc -``` - ---- - -## Partition plan topic (`ranger_audit_partition_plan`) - -Why a **separate** topic from `ranger_audits`: - -- Audit topic carries high-volume events; plan is low-volume config. -- Compacted config topic gives durable “latest value per key” semantics (like Kafka’s own `__consumer_offsets` pattern). - -Suggested topic config: - -```properties -partitions=1 -cleanup.policy=compact -min.compaction.lag.ms=0 # optional: faster visibility of new plan -``` - -**Key:** `ranger_audits` (audit topic name — allows multiple audit topics in future) -**Value:** JSON `PartitionPlan` - -When `dynamic.enabled=true`, each ingestor pod ensures this topic exists at startup via **`createPartitionPlanTopicIfNotExists()`** (implemented in `AuditMessageQueueUtils`; same AdminClient pattern as `createAuditsTopicIfNotExists()` for `ranger_audits`). If the topic is missing, the pod creates it with **1 partition** and **`cleanup.policy=compact`**. - ---- - -## Compacted topic semantics: append, retention, and performance - -This section documents how plan **writes** and **reads** behave on `ranger_audit_partition_plan` — use it when reviewing or extending the design (Phase 2 registry, Phase 3 watcher). - -### Does `writePlan` update the same record? - -**No — Kafka does not update records in place.** Each `writePlan` (REST POST onboard / PATCH update, bootstrap publish) **appends a new record** to partition 0: - -```text -offset 10: key=ranger_audits value=plan v1 -offset 11: key=ranger_audits value=plan v2 -offset 12: key=ranger_audits value=plan v3 -``` - -| Concept | Meaning | -|---------|---------| -| **Physical write** | Always **append** (new offset) | -| **Logical update** | Same **key** (`ranger_audits`) + compaction → only **latest value per key** matters | -| **`version` in JSON** | Application-level optimistic locking (REST `expectedVersion`); not a Kafka in-place edit | - -With `cleanup.policy=compact`, the log cleaner eventually **removes older records for the same key** and retains the **latest** value. After compaction, you typically have **~one record per audit topic key**, not an ever-growing history. - -### Phase 2 read strategy (`KafkaPartitionPlanRegistry.readPlan`) - -Current implementation (admin/bootstrap reads — **not** the audit hot path): - -1. Assign partition 0, `seekToBeginning`. -2. Poll until empty; for each record with matching key, parse JSON into `latest`. -3. Return **last matching record** (highest offset) — correct even if compaction has not run yet. - -This is intentionally simple for **rare** reads (startup, REST GET with force-read, compare-and-swap before POST/PATCH produce). - -### Will growing records hurt performance? - -**Not in normal Ranger use.** This topic is unlike `ranger_audits`: - -| Dimension | `ranger_audit_partition_plan` | `ranger_audits` | -|-----------|-------------------------------|-----------------| -| Write rate | Rare (ops: bootstrap, onboard, update) | Very high (every audit) | -| Typical keys | **1** (`ranger_audits`) | N/A (data topic) | -| Records retained (after compaction) | **~1 per key** | Full retention window | -| Audit POST hot path | **Does not read** this topic | Produces every audit | - -**Cost of `readPlan` full scan** ≈ number of records still on disk for that key before compaction: - -```text -~100 uncompacted versions × ~2 KB JSON ≈ 200 KB → usually well under 100 ms -``` - -Audit ingest throughput is **unaffected** — `AuditPartitioner` reads an in-memory plan only. - -### When record growth could matter (edge cases) - -| Situation | Risk | Mitigation | -|-----------|------|------------| -| Automation hammers PATCH/POST mutations (many versions per minute) | Full scan reads more uncompacted records | Rate-limit ops; rely on compaction; Phase 3 watcher uses **incremental** consume | -| Compaction lag / misconfigured topic | Old versions accumulate on disk | Monitor compaction; set `min.compaction.lag.ms` if needed; verify `cleanup.policy=compact` | -| Many independent audit topic keys on one partition | More keys × versions before compact | Unusual today; one key per deployment is the default design | -| Using full `seekToBeginning` on every watcher poll (anti-pattern) | Unnecessary IO every 30s | Phase 3: consume **new offsets only** after initial load | - -### Phase 3 watcher (planned — do not full-scan every refresh) - -| Phase | Read pattern | When | -|-------|--------------|------| -| **Phase 2** (`readPlan`) | Full compacted log scan | Rare: bootstrap, REST, admin debug | -| **Phase 3** (`PartitionPlanWatcher`) | Initial load once; then **incremental** consumer poll | Every `refresh.interval.ms` (default 30s) | - -Watcher should **not** repeat `seekToBeginning` on every refresh. After startup, track consumer offset and apply only **new** plan records. - -### Ops guardrails (design assumptions) - -- Plan changes are **infrequent** (human or CI), not per-request. -- Default deployment: **one** compacted key (`ranger_audits`) on **one** partition. -- Runtime routing uses **memory**; Kafka is control plane only. -- If plan update frequency ever approaches “every few seconds,” revisit watcher incremental consume and compaction tuning — not audit producer tuning. - -**Implementation reference:** `KafkaPartitionPlanRegistry` in `audit-ingestor/.../partition/` (Phase 2). See [README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md](README-KAFKA-PARTITION-PLAN-IMPLEMENTATION.md) Phase 2–3. - ---- - -## Partition plan JSON (canonical compacted-topic value) - -The compacted topic stores **one JSON document per audit topic key** — nothing else. This is the **only** durable source of truth when `dynamic.enabled=true`. - -**Key:** `ranger_audits` (audit topic name) -**Value:** JSON `PartitionPlan` (fields below — no extra metadata required): - -```json -{ - "topic": "ranger_audits", - "version": 5, - "topicPartitionCount": 28, - "updatedAt": "2026-06-02T12:00:00Z", - "updatedBy": "admin@example.com", - "plugins": { - "hdfs": { "partitions": [0, 1, 2, 3] }, - "hiveServer2": { "partitions": [4, 5, 6, 7, 8, 9] }, - "trino": { "partitions": [10, 11, 12, 13, 14, 15, 16, 17, 18] } - }, - "buffer": { - "partitions": [19, 20, 21, 22, 23, 24, 25, 26, 27] - } -} -``` - -| Field | Purpose | -|-------|---------| -| `topic` | Audit topic this plan applies to (matches compacted topic **key**) | -| `version` | Monotonic integer; optimistic locking on REST POST/PATCH mutations | -| `topicPartitionCount` | Must match Kafka partition count for `topic` | -| `updatedAt` / `updatedBy` | Audit trail (optional but recommended) | -| `plugins` | Map plugin id (`agentId`) → explicit partition ID list | -| `buffer` | Partition IDs for unknown / unconfigured plugins | - -**Create or update:** only via **REST** (`POST .../plugins` / `PATCH .../plugins/{pluginId}`) or **one-time bootstrap publish** when the topic is empty (see below). Producers and audit POST handlers **never** write to this topic. - -**Routing rules in `AuditPartitioner`:** - -1. Lookup `agentId` in `plan.plugins`. -2. If found → round-robin across that plugin’s partition list. -3. If not found → hash into `plan.buffer.partitions`. -4. Validate all partition IDs exist in `cluster.partitionsForTopic(topic)` before applying plan. - ---- - -## Bootstrap and multi-pod startup - -When `dynamic.enabled=true`, **Kafka compacted topic wins** over XML whenever a plan message already exists. - -### First pod: XML populates the plan topic - -On the **first ingestor startup** with dynamic mode enabled, if `ranger_audit_partition_plan` has **no message** for key `ranger_audits`, the first pod **builds v1 from XML and publishes it** to the compacted topic. That is how the plan registry is initially filled. - -#### When bootstrap-from-XML runs - -| Condition | Bootstrap from XML → plan topic? | -|-----------|----------------------------------| -| `dynamic.enabled=true` **and** plan topic empty | **Yes** — first pod publishes v1 | -| `dynamic.enabled=true` **and** plan already in Kafka | **No** — read Kafka only | -| `dynamic.enabled=false` or property absent | **No** — legacy XML at startup; plan topic unused | - -#### XML properties used to build v1 - -| XML property | Role in v1 plan | -|--------------|-----------------| -| `ranger.audit.ingestor.kafka.configured.plugins` | Plugin ids → keys in `plan.plugins` (order defines initial contiguous ranges) | -| `ranger.audit.ingestor.kafka.plugin.partition.overrides.` | Partition count for that plugin (length of its `partitions` list) | -| `ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin` | Default count per plugin when no override (typically **3**) | -| `ranger.audit.ingestor.kafka.topic.partitions.buffer` | Size of `plan.buffer.partitions` (typically **9**) | - -The bootstrap step **converts** today’s contiguous-range layout into **explicit partition ID lists** in JSON (same shape as REST-managed plans). - -#### Example: XML → v1 in `ranger_audit_partition_plan` - -**XML (simplified):** `configured.plugins=hdfs,hiveServer2`, default 3 partitions each, buffer 9. - -**Kafka compacted topic** — key `ranger_audits`, value: - -```json -{ - "topic": "ranger_audits", - "version": 1, - "topicPartitionCount": 15, - "updatedAt": "2026-06-02T10:00:00Z", - "updatedBy": "bootstrap", - "plugins": { - "hdfs": { "partitions": [0, 1, 2] }, - "hiveServer2": { "partitions": [3, 4, 5] } - }, - "buffer": { - "partitions": [6, 7, 8, 9, 10, 11, 12, 13, 14] - } -} -``` - -After this one-time publish, **all routing and later changes** use the plan in Kafka (REST onboard/update), not XML edits. - -```mermaid -flowchart LR - XML[ranger-audit-ingestor-site.xml] -->|first pod only| Build[Build v1] - Build --> Publish[Publish to ranger_audit_partition_plan] - Publish --> Mem[Load into memory] - Mem --> Audits[Serve audits] -``` - ---- - -### Pod 1 (first ingestor, empty plan topic) - -**Description:** First pod with `dynamic.enabled=true` creates plan topic (if needed), seeds **v1 from XML** into Kafka, loads into memory. XML is used **once**. - -| Step | Action | -|------|--------| -| 1 | `PartitionPlanWatcher` starts | -| 2 | `createPlanTopicIfNotExists()` — idempotent ([Race A](#race-a--multiple-pods-create-the-plan-topic-at-the-same-time)) | -| 3 | Read plan topic → no message for key `ranger_audits` | -| 4 | Build v1 from XML → explicit partition lists | -| 5 | Publish v1 to `ranger_audit_partition_plan` ([Race B](#race-b--multiple-pods-publish-plan-v1-when-topic-exists-but-has-no-message)) | -| 6 | Re-read compacted topic → install v1 in memory | -| 7 | Serve audits (memory only on hot path) | - -```mermaid -flowchart TD - Start[Pod 1 starts] --> Watcher[PartitionPlanWatcher] - Watcher --> Create[createPlanTopicIfNotExists] - Create --> Read{Plan message exists?} - Read -->|No| XML[Build v1 from XML] - XML --> Publish[Publish v1 to plan topic] - Publish --> Reread[Re-read compacted topic] - Reread --> Mem[Install in memory] - Mem --> Audits[Serve POST /access] -``` - -**Alternative:** wait for first admin POST onboard instead of auto-publish — possible, but auto-publish is recommended for pod 2+. - -### Pod 2+ (later ingestors, plan already in Kafka) - -**Description:** Plan already in Kafka — **read only**, never re-bootstrap from XML for routing. - -| Step | Action | -|------|--------| -| 1 | `PartitionPlanWatcher` starts | -| 2 | `createPlanTopicIfNotExists()` — usually no-op (topic exists) | -| 3 | Read plan version **N** from compacted topic | -| 4 | Install into memory — **ignore XML** | -| 5 | Serve audits | - -```mermaid -flowchart TD - Start[Pod 2+ starts] --> Watcher[PartitionPlanWatcher] - Watcher --> Create[createPlanTopicIfNotExists no-op] - Create --> Read[Read plan vN from Kafka] - Read --> Mem[Install in memory] - Mem --> Audits[Serve POST /access] - XML[XML config] -.->|not used for routing| Mem -``` - -**Do not** bootstrap-from-XML again if a plan message exists — avoids ConfigMap drift across pods. - ---- - -### Race A — multiple pods create the plan **topic** at the same time - -Several ingestor pods can start together when `ranger_audit_partition_plan` **does not exist yet**. Each pod may call `listTopics()`, see nothing, and call `createTopics()` — same class of race as `ranger_audits` creation in `AuditMessageQueueUtils.createAuditsTopicIfNotExists()`. - -**Required behavior (idempotent topic create):** - -**Description:** Multiple pods may call `createTopics()` at once. **Already exists = success.** Only fail on real errors (ACL, Kafka down). - -| Step | Action | -|------|--------| -| 1 | `listTopics()` — if topic exists → OK, continue | -| 2 | `createTopics(1 partition, cleanup.policy=compact)` | -| 3 | Success → `waitUntilTopicReady` → continue | -| 4 | Failure **already exists** → treat as success, `describe` topic, continue | -| 5 | Other failure → fail startup with clear error | - -```mermaid -flowchart TD - Start[createPlanTopicIfNotExists] --> List{Topic exists?} - List -->|Yes| OK[Return OK] - List -->|No| Create[createTopics] - Create --> Success{Result?} - Success -->|OK| Wait[waitUntilTopicReady] - Wait --> OK - Success -->|Already exists| OK - Success -->|Other error| Fail[Fail startup] -``` - -| Outcome | Pod behavior | -|---------|----------------| -| This pod creates topic | Log info, proceed to read/publish plan | -| Peer created topic first | Catch already-exists, describe topic, proceed | -| Create fails for other reason | Fail startup (Kafka unreachable, ACL denied) | - -```mermaid -sequenceDiagram - participant PodA as ingestor pod A - participant PodB as ingestor pod B - participant Kafka as Kafka Admin - - PodA->>Kafka: listTopics — plan topic missing - PodB->>Kafka: listTopics — plan topic missing - PodA->>Kafka: createTopics(ranger_audit_partition_plan) - PodB->>Kafka: createTopics(ranger_audit_partition_plan) - Kafka-->>PodA: OK - Kafka-->>PodB: already exists - PodB->>PodB: treat as success, continue - Note over PodA,PodB: both proceed to plan read / bootstrap -``` - -**Implementation note:** reuse or factor shared logic from `audit-common/.../AuditMessageQueueUtils.java` (`createAuditsTopicIfNotExists`) and add explicit **already-exists** handling if not present today (concurrent `createTopics` on `ranger_audits` has the same gap). - ---- - -### Race B — multiple pods publish plan **v1** when topic exists but has no message - -**Description:** Topic exists but no plan message yet. Both pods may build v1 — **re-read before/after produce**; mandatory read-back; adopt Kafka’s plan. - -| Step | Action | -|------|--------| -| 1 | Both pods: `ensurePlanTopicExists()` (Race A done) | -| 2 | Read plan → null | -| 3 | Build v1 from XML | -| 4 | **Re-read** — if v1 appeared → install, skip publish | -| 5 | Produce local v1 (first wins on offset) | -| 6 | **Mandatory re-read** → install stored plan (not local object only) | - -```mermaid -flowchart TD - Start[bootstrapPlanIfEmpty] --> Ensure[Race A: topic exists] - Ensure --> Read1[Read plan from Kafka] - Read1 -->|plan exists| Install1[Install and return] - Read1 -->|empty| Build[Build v1 from XML] - Build --> Read2[Re-read plan] - Read2 -->|peer published| Install2[Install peer plan return] - Read2 -->|still empty| Produce[Produce local v1] - Produce --> Read3[Mandatory read-back] - Read3 --> Install3[Install Kafka plan] -``` - -```mermaid -sequenceDiagram - participant A as pod A - participant B as pod B - participant Plan as ranger_audit_partition_plan - - A->>Plan: read → empty - B->>Plan: read → empty - A->>Plan: produce v1 - B->>Plan: produce v1 or re-read sees v1 - A->>Plan: read-back → install v1 - B->>Plan: read-back → install same v1 - Note over A,B: both pods same routing -``` - -This preserves backward compatibility with `ranger-audit-ingestor-site.xml` for greenfield installs. - ---- - -## Kafka read load (REST + background thread only) - -The plan topic is **low volume** (config, not audit events). Avoid reading it on the hot audit path. - -| Code path | Reads plan topic? | Notes | -|-----------|-------------------|--------| -| Plugin audit POST → `AuditMessageQueue` → `AuditPartitioner.partition()` | **No** | Reads `AtomicReference` in memory only | -| `PartitionPlanWatcher` (background thread) | **Yes** | Poll / consume on interval or on compacted message | -| REST `GET /api/audit/partition-plan` | **Yes** | Admin read; may return cached in-memory plan if fresh | -| REST POST/PATCH plugins | **Yes** | Read current version, then produce new version | -| Audit produce to `ranger_audits` | **No** | Unrelated topic | - -**Design rule:** at most **one background consumer per ingestor pod** plus **occasional REST reads/writes**. No per-request Kafka access for partition routing. - -Recommended watcher behavior: - -- **Startup:** one read to load initial plan (from compacted topic, or bootstrap-then-read). -- **Runtime:** consumer poll with long interval (`refresh.interval.ms`, default 30s) **or** event-driven on new compacted message — not tight spin loops. -- **On failure:** keep **last known good** plan in memory; do not block audit POST. - -With a single-partition compacted topic and ~30s refresh, Kafka load from N ingestor pods is negligible compared to `ranger_audits` traffic. - -**Append vs update:** each plan write adds a record; compaction + “latest per key” semantics mean the effective store is one current plan. See [Compacted topic semantics: append, retention, and performance](#compacted-topic-semantics-append-retention-and-performance). - ---- - -## REST API (control plane) - -The partition-plan admin API exposes **three endpoints**. All mutation requests use optimistic locking via `expectedVersion` (current plan version from `GET`). Reject with `409 Conflict` when stale; response body includes the current plan. - -**Auth:** Kerberos/JWT + admin role (not the same as plugin audit POST users). When `kafka.partition.plan.allowed.users` is configured, only those short names may call these endpoints. - -### GET `/api/audit/partition-plan` - -Returns the current plan from the in-memory holder (same JSON shape as the compacted Kafka value). - -### POST `/api/audit/partition-plan/plugins` — onboard plugin - -Onboards a plugin from the buffer to dedicated partitions and registers service allowlists in **one** plan version bump. - -**Required fields:** `pluginId`, `partitionCount`, `expectedVersion`, `services` - -**`services` map:** repo name → `{ "allowedUsers": [...] }`. Must be non-empty. Each entry is tagged with `pluginId` in the stored plan. - -```json -{ - "pluginId": "hiveServer2", - "partitionCount": 3, - "expectedVersion": 1, - "services": { - "dev_hive": { "allowedUsers": ["hive"] }, - "dev_hive2": { "allowedUsers": ["hive2"] } - } -} -``` - -Server: takes partition IDs from buffer (or grows `ranger_audits` tail), adds plugin assignment, merges services with `pluginId`, writes plan v(N+1). - -### PATCH `/api/audit/partition-plan/plugins/{pluginId}` — update plugin - -Updates an onboarded plugin: scale tail partitions and/or mutate service allowlists scoped to `{pluginId}` in one version bump. - -**Required:** `expectedVersion` - -**At least one of:** - -| Field | Purpose | -|-------|---------| -| `additionalPartitions` | int ≥ 1 — append tail partition IDs (same append-only semantics as legacy scale) | -| `addServices` | map repo → `{ "allowedUsers": [...] }` — add repos tagged with path `pluginId` | -| `updateServices` | map repo → `{ "allowedUsers": [...] }` — replace allowlist for repos owned by path `pluginId` (or legacy entries with no `pluginId`) | -| `removeServices` | list of repo names — remove allowlist entries owned by path `pluginId` (or legacy entries with no `pluginId`) | - -**Example — Hive multi-repo lifecycle** (onboard two repos, add a third, remove one, scale): - -```bash -# 1) Onboard hiveServer2 with dev_hive + dev_hive2 (plan v1 → v2) -curl -X POST .../api/audit/partition-plan/plugins -d '{ - "pluginId": "hiveServer2", - "partitionCount": 3, - "expectedVersion": 1, - "services": { - "dev_hive": { "allowedUsers": ["hive"] }, - "dev_hive2": { "allowedUsers": ["hive2"] } - } -}' - -# 2) Add dev_hive3, remove dev_hive2 (plan v2 → v3) -curl -X PATCH .../api/audit/partition-plan/plugins/hiveServer2 -d '{ - "expectedVersion": 2, - "addServices": { "dev_hive3": { "allowedUsers": ["hive3"] } }, - "removeServices": ["dev_hive2"] -}' - -# 3) Scale +2 tail partitions (plan v3 → v4) -curl -X PATCH .../api/audit/partition-plan/plugins/hiveServer2 -d '{ - "expectedVersion": 3, - "additionalPartitions": 2 -}' -``` - -Stored service entries include optional `"pluginId": "hiveServer2"` for ownership tracking. - -**Removed endpoints** (consolidated above): `PATCH /api/audit/partition-plan`, `POST /api/audit/partition-plan/services`, and separate promote-only / scale-only semantics. - -> **Future work (not in this change):** bootstrap **v0** split — seeding `services` from XML separately from plugin partition assignments. Current bootstrap still publishes a single v1 plan from XML as today. - ---- - -## Mutation handler flow (POST onboard / PATCH update) - -**Description:** Admin POST/PATCH on any ingestor pod. Grow `ranger_audits` **before** publishing plan that references new partition IDs. Server assigns `version = N+1`; client sends `expectedVersion` only. - -| Step | Action | -|------|--------| -| 1 | Authenticate + authorize admin | -| 2 | Load plan from compacted topic (version **N**) | -| 3 | If `expectedVersion != N` → **409** + current plan | -| 4 | Validate + allocate (onboard: partitions + services; update: scale and/or service mutations) | -| 5 | If needed → `AdminClient.createPartitions(ranger_audits)` | -| 6 | **Re-read** plan — still **N**? else **409** | -| 7 | Produce plan **N+1** to plan topic | -| 8 | **Read-back** — matches intent? else **409** | -| 9 | Return **200** + full plan | - -```mermaid -flowchart TD - Admin[Admin POST/PATCH] --> Auth[Auth + authorize] - Auth --> Load[Load plan vN] - Load --> Check1{expectedVersion = N?} - Check1 -->|No| R409a[409 + current plan] - Check1 -->|Yes| Valid[Validate + allocate] - Valid --> Grow{Need more partitions?} - Grow -->|Yes| CP[createPartitions ranger_audits] - Grow -->|No| Reread[Re-read plan] - CP --> Reread - Reread --> Check2{Still vN?} - Check2 -->|No| R409b[409 peer won] - Check2 -->|Yes| Produce[Produce vN+1] - Produce --> Back[Read-back verify] - Back --> Check3{Matches intent?} - Check3 -->|No| R409c[409 lost race] - Check3 -->|Yes| OK[200 + plan] -``` - -**Order matters:** increase audit topic **before** plan references new IDs. - -**Version is server-assigned** — clients must not set the new version number. - ---- - -## Concurrent updates from multiple ingestor pods - -REST is exposed on **every** ingestor pod behind a load balancer. Two admins (or automation jobs) can hit **different pods at the same time**. Writes must not double-allocate partition IDs or overwrite each other silently. - -### Recommended approach: optimistic concurrency + compare-and-swap - -This is the **default** implementation — no leader election required. - -| Mechanism | Role | -|-----------|------| -| **`expectedVersion` in request** | Client declares which plan it based its change on | -| **Early reject** | If compacted topic already at version ≠ `expectedVersion` → **409 Conflict** before allocation | -| **Re-read before produce** | After `createPartitions` (if any), re-read topic; if version changed → **409** without producing | -| **Single-partition plan topic** | All plan writes are totally ordered by Kafka offset | -| **Read-back after produce** | Confirm the compacted value is the plan this handler intended; else **409** | -| **Client retry** | On 409: `GET` latest plan, re-apply intent with new `expectedVersion` | - -```mermaid -sequenceDiagram - participant AdminA as Admin / job A - participant PodA as ingestor pod A - participant PodB as ingestor pod B - participant PlanTopic as ranger_audit_partition_plan - - Note over PlanTopic: current version = 4 - - AdminA->>PodA: PATCH .../plugins/hiveServer2 expectedVersion=4 - AdminA->>PodB: POST .../plugins expectedVersion=4 - - PodA->>PlanTopic: read plan → v4 - PodB->>PlanTopic: read plan → v4 - - PodA->>PodA: allocate + createPartitions if needed - PodB->>PodB: allocate + createPartitions if needed - - PodA->>PlanTopic: re-read → still v4 - PodB->>PlanTopic: re-read → still v4 - - PodA->>PlanTopic: produce v5 (update hiveServer2) - PlanTopic-->>PodB: compacted update visible - - PodB->>PlanTopic: re-read → now v5 - PodB-->>AdminA: 409 Conflict + current plan v5 - - Note over AdminA: retry with expectedVersion=5 - - AdminA->>PodB: POST .../plugins expectedVersion=5 - PodB->>PlanTopic: read v5 → allocate → produce v6 - PodB-->>AdminA: 200 OK plan v6 -``` - -### What each outcome means - -| HTTP | Meaning | Client action | -|------|---------|---------------| -| **200** | This handler’s plan v(N+1) is in the compacted topic | Done | -| **409 Conflict** | Stale `expectedVersion` or lost compare-and-swap race | `GET` plan, merge intent, retry with new `expectedVersion` | -| **503** | `createPartitions` failed or Kafka produce failed | Retry same request (idempotent if version unchanged) | - -**409 response body** should include the **current plan** (version, plugins, buffer) so callers can retry without a separate GET. - -### Why this is safe for partition allocation - -Two pods both scaling from v4 might temporarily compute overlapping tail IDs in memory — only one produce wins the compare-and-swap. The loser gets **409** and must **re-read v5** and allocate from the **new tail**, not reuse its stale allocation. Append-only validation on POST/PATCH rejects plans that steal IDs already assigned in the current version. - -### Residual race: both produce before either re-reads - -**Description:** Plan topic has **one partition** — Kafka orders writes by offset. Only one v5 survives in compaction; loser returns **409**, client retries on winning plan. - -| Step | What happens | -|------|----------------| -| 1 | Pod A and B both pass re-read at v4 | -| 2 | Both produce v5 — higher **offset wins** in compacted topic | -| 3 | Loser read-back ≠ intended plan → **409** to client (not 200) | -| 4 | All watchers load **single** winning v5 | - -```mermaid -sequenceDiagram - participant A as pod A handler - participant B as pod B handler - participant Plan as plan topic 1 partition - - Note over Plan: both saw v4 at re-read - A->>Plan: produce v5 offset 100 - B->>Plan: produce v5 offset 101 - Note over Plan: compaction keeps offset 101 - A->>Plan: read-back - A-->>A: content != intent → 409 - B->>Plan: read-back - B-->>B: matches → 200 - Note over Plan: watchers all load one v5 -``` - -No silent split-brain — one plan version in registry; loser retries with `expectedVersion=5`. - -### Optional: stricter serialization (if you want zero client retry) - -| Approach | When to use | -|----------|-------------| -| **K8s Lease** (`partition-plan-writer`) | Only the lease holder executes PATCH/POST mutations; other pods return **503 Retry on leader** or proxy internally | -| **Single admin job / CI** | Human or pipeline serializes changes — sufficient for many ops teams | -| **Dedicated single-replica “admin” ingestor** | Extreme isolation; usually unnecessary | - -Start with **optimistic concurrency + compare-and-swap + client retry**. Add leader lease only if automation cannot tolerate 409 retries. - -### What ingestor pods do *not* do - -- **No cross-pod locking** on the audit POST path. -- **No per-pod plan writes** from the watcher (watchers are **read-only**). -- **No merge of concurrent admin intent on the server** — conflicts are explicit 409; the client or operator merges. - ---- - -## How every ingestor pod stays in sync - -**Description:** Each pod runs **`PartitionPlanWatcher`** (background only). `AuditPartitioner` reads memory — never plan topic on audit POST. - -| Option | Behavior | -|--------|----------| -| **A — Consumer (recommended)** | Group `ranger_audit_partition_plan_watcher`; on message → validate → `partitionPlanRef.set()` | -| **B — Periodic poll** | Every `refresh.interval.ms` (default 30s) read latest compacted value | - -```mermaid -flowchart TB - Plan[(ranger_audit_partition_plan)] - - subgraph pod1 [Ingestor pod 1] - W1[PartitionPlanWatcher] - M1[(Memory plan)] - P1[AuditPartitioner] - W1 --> M1 --> P1 - end - - subgraph pod2 [Ingestor pod 2] - W2[PartitionPlanWatcher] - M2[(Memory plan)] - P2[AuditPartitioner] - W2 --> M2 --> P2 - end - - Plan --> W1 - Plan --> W2 - - Plugin1[Plugin POST] --> P1 - Plugin2[Plugin POST] --> P2 -``` - -**Latency:** all pods converge within ~30s or on Kafka message. **No restart** when plan changes. - ---- - -## Pod crash / restart - -**Description:** Same as **Pod 2+** — memory lost on crash; plan survives in Kafka compacted topic. - -| Step | Action | -|------|--------| -| 1 | Pod dies — in-memory plan lost | -| 2 | New pod starts | -| 3 | Watcher reads plan **vN** from Kafka | -| 4 | Install memory — routing restored | - -```mermaid -flowchart LR - Crash[Pod crash] --> Lost[Memory lost] - Lost --> New[New pod starts] - New --> Read[Read plan vN from Kafka] - Read --> Mem[Install memory] - Mem --> OK[Same routing as other pods] -``` - -Plan is **not** on pod disk — Kafka compacted topic is durable store. - ---- - -## Worked example: hdfs + hiveServer2, then onboard trino - -**Initial plan (v1)** — from bootstrap: - -| Plugin | Partitions | -|--------|------------| -| hdfs | [0,1,2] | -| hiveServer2 | [3,4,5] | -| buffer | [6..14] (9 partitions) | - -Topic: **15** partitions. - -**Unknown plugin `trino` sends audits** → routes to buffer (hash among [6..14]). - -**Admin: POST .../plugins — onboard trino (partitionCount=3, mandatory services)** - -```json -{ - "pluginId": "trino", - "partitionCount": 3, - "expectedVersion": 1, - "services": { - "dev_trino": { "allowedUsers": ["trino"] } - } -} -``` - -1. Allocator takes IDs [6,7,8] from buffer (or adds tail if buffer policy prefers grow-first). -2. buffer becomes [9..14]; `dev_trino` allowlist merged with `pluginId: trino`. -3. Plan v2 published. -4. All ingestors pick up v2 within refresh interval. -5. **No ingestor restart.** - -**Later: PATCH .../plugins/hiveServer2 — additionalPartitions=3** - -1. Increase topic 15 → 18 if needed. -2. Append partition IDs [15,16,17] to hiveServer2 list (append-only — hdfs and trino lists unchanged). -3. Plan v3 published. - -This fixes the “change hdfs override reshuffles everyone” problem from static contiguous allocation. - ---- - -## Solr / HDFS dispatchers - -Dispatchers (`README-KAFKA-DISPATCHERS.md`) **ignore** the partition plan: - -- They subscribe to **all** partitions of `ranger_audits`. -- When audit topic grows 15 → 18, both consumer groups rebalance and assign new partitions. -- No dispatcher code or config change for plugin onboarding. - -You may need to scale dispatcher `thread.count` / replicas if partition count grows significantly. - ---- - -## Failure modes and handling - -| Failure | Behavior | -|---------|----------| -| Plan topic unreadable | Keep **last known good** plan in memory; log error; `/status` shows degraded | -| Invalid plan JSON | Reject write at REST; do not apply on read | -| AdminClient partition increase fails | Do **not** publish new plan; return 503 to caller | -| Partial write (partitions increased but plan not written) | Retry POST/PATCH idempotently; allocator sees higher topic count | -| Two concurrent POST/PATCH (different pods) | `expectedVersion` + re-read before produce + read-back → one **200**, one **409**; client retries on latest version | -| Plugin sends audits during plan swap | `AtomicReference` swap is atomic; brief mix of old/new routing acceptable for audits | - ---- - -## Feature flag: backward compatibility (old vs dynamic behavior) - -**Description:** Dynamic mode is **opt-in**. Property **missing** or **`false`** → legacy behavior (today). **`true`** → Kafka plan topic + watcher + REST. - -Dynamic mode is **disabled** when: - -- `ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled` is **`false`**, or -- the property is **missing / not set** (default = **false**) - -```mermaid -flowchart TD - Flag{dynamic.enabled?} - Flag -->|false or absent| Legacy[Legacy mode] - Flag -->|true| Dynamic[Dynamic mode] - - Legacy --> L1[AuditPartitioner from XML at startup] - Legacy --> L2[No plan topic / no watcher] - Legacy --> L3[XML change + restart] - - Dynamic --> D1[Plan from Kafka compacted topic] - Dynamic --> D2[Watcher + REST enabled] - Dynamic --> D3[XML bootstrap once if empty] -``` - -### When dynamic mode is OFF (default — today’s behavior) - -| Area | Behavior | -|------|----------| -| Partition mapping | `AuditPartitioner.configure()` at startup from XML: `configured.plugins`, overrides, contiguous ranges + buffer | -| Topic partitions | Created/updated via `AuditMessageQueueUtils.getPartitions()` from XML (sum + buffer) | -| Changes | Update XML → restart audit-ingestor | -| REST `/api/audit/partition-plan` | **Not registered** or returns `404` / `503 Feature disabled` | -| `PartitionPlanWatcher` | **Not started** | -| Plan topic `ranger_audit_partition_plan` | **Not used** (may be omitted from cluster) | - -This preserves **100% backward compatibility** for existing deployments with no config change. - -### When dynamic mode is ON (`dynamic.enabled=true`) - -| Area | Behavior | -|------|----------| -| Partition mapping | Runtime `PartitionPlan` from Kafka compacted topic (XML seeds bootstrap only if plan topic empty) | -| Topic partitions | AdminClient increase driven by plan / REST | -| Changes | REST → Kafka plan topic; all pods refresh via watcher — **no restart** | -| REST partition-plan endpoints | **Enabled** (admin AuthZ) | -| `PartitionPlanWatcher` | **Started** on every ingestor pod | - -### Recommended default in code and sample XML - -```xml - - ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled - false - - Enable Kafka-backed dynamic partition plan (registry topic + REST + watcher). - Default false: use legacy XML-based AuditPartitioner behavior at startup. - If property is absent, treat as false. - - -``` - -Only set to `true` after `ranger_audit_partition_plan` topic exists and ops are ready to manage plans via REST. - -### Property defaults when `dynamic.enabled=true` - -If dynamic mode is on, missing optional properties use **built-in defaults** (do not fall back to legacy XML routing): - -| Property | If absent | Effective value | -|----------|-----------|-----------------| -| `partition.plan.topic` | Use default | **`ranger_audit_partition_plan`** | -| `partition.plan.refresh.interval.ms` | Use default | **`30000`** (30 seconds) | - -**Startup behavior with defaults:** - -**Description:** When `dynamic.enabled=true`, optional properties default as below. Fail startup if Kafka/plan topic unavailable — do not silently fall back to legacy XML. - -| Step | Action | -|------|--------| -| 1 | Resolve plan topic → **`ranger_audit_partition_plan`** (unless overridden) | -| 2 | **`createPlanTopicIfNotExists()`** — idempotent ([Race A](#race-a--multiple-pods-create-the-plan-topic-at-the-same-time)) | -| 3 | Start **`PartitionPlanWatcher`** | -| 4 | If no plan message → bootstrap v1 from XML ([Race B](#race-b--multiple-pods-publish-plan-v1-when-topic-exists-but-has-no-message)) | - -```mermaid -flowchart TD - Start[Ingestor startup dynamic=true] --> Topic[Resolve plan topic name] - Topic --> Create[createPlanTopicIfNotExists Race A] - Create --> Watcher[Start PartitionPlanWatcher] - Watcher --> Empty{Plan message empty?} - Empty -->|Yes| Boot[Bootstrap v1 from XML Race B] - Empty -->|No| Load[Load plan from Kafka] - Boot --> Ready[Ready serve audits] - Load --> Ready -``` - -**You do not need** to set `partition.plan.topic` unless using a non-default name. - -**Misconfiguration:** If `dynamic.enabled=true` but Kafka unreachable → **fail startup** (avoid split routing across pods). - ---- - -## What stays in XML (bootstrap only) - -| Property | Role after dynamic mode | -|----------|-------------------------| -| `configured.plugins` | Initial plan seed only (first pod bootstrap when plan topic empty) | -| `plugin.partition.overrides.*` | Initial counts when seeding v1 | -| `topic.partitions.buffer` | Initial buffer list size when seeding v1 | -| `topic.partitions.per.configured.plugin` | Default per-plugin count when seeding v1 (if no override) | -| `partitioner.class` | Still `AuditPartitioner` at JVM startup | -| `partition.plan.topic` | Compacted registry topic name; **default `ranger_audit_partition_plan`** if unset | -| `partition.plan.dynamic.enabled` | **`false` by default** — set `true` to enable dynamic mode | - -When `dynamic.enabled` is **false** or **unset**, XML properties above drive **legacy startup behavior** (not bootstrap-only). - -Runtime changes (when dynamic enabled) go through **REST → Kafka plan topic**, not XML edits. - ---- - -## Bootstrap configuration (new properties) - -```xml - - - ranger.audit.ingestor.kafka.partition.plan.topic - ranger_audit_partition_plan - - - ranger.audit.ingestor.kafka.partition.plan.refresh.interval.ms - 30000 - - - ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled - false - Default false. Set true to enable dynamic partition plan. If absent, use legacy XML behavior. - -``` - -Existing `configured.plugins` / overrides remain **bootstrap** when the compacted topic has no plan yet. - ---- - -## Implementation order (recommended) - -**Description:** Build bottom-up — model and registry first, then partitioner + watcher, then REST and ops. - -| Phase | Deliverable | -|-------|-------------| -| 1 | `PartitionPlan` model + allocator tests (append-only) | -| 2 | Registry read/write + `createPlanTopicIfNotExists` (Race A) | -| 3 | Plan-aware `AuditPartitioner` + watcher + bootstrap (Race B) | -| 4 | REST GET (inspect plan) | -| 5 | REST GET / POST / PATCH plugins | `PartitionPlanService` | -| 6 | AuthZ + `/status` (`plan.version`, watcher timestamp) | -| 7 | Ops runbook — REST only when dynamic=true → [README-KAFKA-PARTITION-PLAN-OPS-RUNBOOK.md](README-KAFKA-PARTITION-PLAN-OPS-RUNBOOK.md) | -| 8 | Migrate deployments — publish v1 from current XML | - -```mermaid -flowchart LR - P1[1 Model + allocator] --> P2[2 Registry + topic create] - P2 --> P3[3 Partitioner + watcher] - P3 --> P4[4 REST GET] - P4 --> P5[5 REST POST/PATCH plugins] - P5 --> P6[6 AuthZ + status] - P6 --> P7[7 Runbook] - P7 --> P8[8 Migration v1] -``` - ---- - -## Design rationale and tradeoffs - -**Assessment:** This is a **sound approach** for Ranger audit-ingestor: Kafka is already required, partition-plan changes are infrequent (ops events, not per-request), and the design keeps the audit hot path off Kafka. It is not the only valid design, but the tradeoffs fit the problem well. - -### Why this design works - -| Principle | How this design delivers | -|-----------|--------------------------| -| **Established pattern** | Kafka compacted topic as config store (same idea as Connect/Streams config topics): one key → latest plan, durable across restarts | -| **Hot path stays fast** | `AuditPartitioner.partition()` reads `AtomicReference` only; plan topic touched by watcher + REST, not every audit POST | -| **Append-only explicit lists** | Avoids reshuffling later plugins when an early plugin scales (main limitation of static contiguous XML allocation) | -| **Multi-pod consistency** | Pod 1 seeds Kafka from XML once; pod 2+ and restarts read Kafka; all replicas converge on the same JSON | -| **Safe rollout** | `dynamic.enabled` defaults to `false` — existing deployments unchanged | -| **Clean consumer boundary** | Solr/HDFS dispatchers consume all partitions; plan is producer-side only | - -### Acceptable tradeoffs - -| Concern | Severity | Mitigation in this design | -|---------|----------|---------------------------| -| Kafka required at startup when dynamic=true | Medium | Fail startup with clear error; do not silently fall back to legacy XML (would split routing across pods) | -| Two pods bootstrap race on empty plan topic | Low | **Race A:** idempotent `createPlanTopicIfNotExists` + already-exists OK; **Race B:** re-read before/after v1 produce | -| Watcher refresh lag (~30s default) | Low | Acceptable for plugin onboarding; tune `refresh.interval.ms` if needed | -| Ops edit XML expecting live effect | Medium | When dynamic=true, runbook: change plan via REST, not XML (XML is bootstrap-only) | -| `createPartitions` succeeds but plan write fails | Medium | Increase audit topic **before** publishing plan; retry POST/PATCH idempotently | -| ConfigMap drift between pods | Low | Kafka is source of truth once v1 exists | - -These are normal distributed-config caveats, not fundamental flaws. - -### When this approach is a poor fit - -Avoid dynamic mode (or reconsider the registry) if: - -- Plan changes happen **very frequently** (seconds) — watcher lag and consumer rebalance churn add cost. -- Kafka is **unavailable or not part of the stack** — use legacy XML or an external config store instead. -- You operate **many independent audit topics** with separate plans — still workable (one compacted key per topic) but REST and ops complexity grow. -- You require **strict per-plugin ordering across plan migrations** — append-only reduces reshuffle risk, but any partition reassignment still needs operational care. - -For typical Ranger deployments (infrequent plugin onboard/scale, Kafka already present), none of these apply strongly. - -### Alternatives considered - -| Option | Tradeoff vs this design | -|--------|-------------------------| -| **XML + restart only** (today) | Simplest; no dynamic onboarding without downtime | -| **Postgres / ZooKeeper** | Stronger CRUD and consistency; extra infra this design deliberately avoids | -| **K8s ConfigMap + watch** | K8s-centric; awkward for mixed environments, multi-cluster, or REST-driven automation | -| **Leader pod writes local XML** | Breaks with multiple replicas — rejected | - -Given **no new infra**, **multi-replica ingestor**, **REST-driven ops**, and **survive pod restart**, Kafka compacted topic + REST + background watcher is a strong fit. - -### Implementation guardrails - -1. **Bootstrap:** `createPlanTopicIfNotExists()` must be idempotent (Race A); after plan publish, always re-read compacted topic (Race B). -2. **GET `/partition-plan`:** prefer in-memory plan for normal reads; optional force-read from Kafka for debugging. -3. **`/status`:** expose `plan.version` and watcher last-updated timestamp. -4. **Mutation validation:** optional strict mode — reject writes that shrink or reassign existing plugin partition IDs. -5. **Runbook:** document that when `dynamic.enabled=true`, runtime changes go through REST, not XML edits. - ---- - -## Summary - -| Question | Answer | -|----------|--------| -| Postgres/ZK needed? | **No** — use Kafka compacted topic + AdminClient | -| Update XML in pod? | **No** for runtime — XML is bootstrap only | -| REST endpoint? | **Yes** — writes plan to Kafka + increases partitions via AdminClient | -| Pod crash/restart? | Re-read plan from Kafka compacted topic | -| Other pods notified? | All pods watch same topic / poll same plan | -| Dispatchers affected? | Rebalance only when partition count grows; no plan API needed | diff --git a/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh deleted file mode 100755 index f33507286e7..00000000000 --- a/dev-support/ranger-docker/scripts/audit/dynamic-auth-to-local-e2e-lib.sh +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# E2E helpers: dynamic allowlist + auth_to_local + POST /api/audit/access via curl/SPNEGO. -# Requires partition-plan-e2e-lib.sh (pp_*) to be sourced first. - -DAEL_INGESTOR_HOST="${CONTAINER}.rangernw" -DAEL_INGESTOR_HTTP_PORT="${INGESTOR_HTTP_PORT:-7081}" -DAEL_ACCESS_PATH="/api/audit/access" -DAEL_ONBOARD_PATH="/api/audit/partition-plan/plugins" -DAEL_ACCESS_CODE="" -DAEL_ACCESS_BODY="" - -# repo|appId|shortName|source_container|keytab|principal -# shortName = auth_to_local output checked against services[].allowedUsers -# appId = Kafka plugin id (agentId on audit events) -readonly -a DAEL_PLUGIN_PROFILES=( - "dev_hdfs|hdfs|hdfs|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" - "dev_yarn|yarn|yarn|ranger-hadoop|/etc/keytabs/yarn.keytab|yarn/ranger-hadoop.rangernw@EXAMPLE.COM" - "dev_hive|hiveServer2|hive|ranger-hive|/etc/keytabs/hive.keytab|hive/ranger-hive.rangernw@EXAMPLE.COM" - "dev_hbase|hbaseMaster|hbase|ranger-hbase|/etc/keytabs/hbase.keytab|hbase/ranger-hbase.rangernw@EXAMPLE.COM" - "dev_kafka|kafka|kafka|ranger-kafka|/etc/keytabs/kafka.keytab|kafka/ranger-kafka.rangernw@EXAMPLE.COM" - "dev_knox|knox|knox|ranger-knox|/etc/keytabs/knox.keytab|knox/ranger-knox.rangernw@EXAMPLE.COM" - "dev_kms|kms|rangerkms|ranger-kms|/etc/keytabs/rangerkms.keytab|rangerkms/ranger-kms.rangernw@EXAMPLE.COM" - "dev_trino|trino|trino|ranger-trino|/etc/keytabs/trino.keytab|trino/ranger-trino.rangernw@EXAMPLE.COM" - "dev_solr|solr|solr|ranger-solr|/etc/keytabs/solr.keytab|solr/ranger-solr.rangernw@EXAMPLE.COM" - "dev_ozone|ozone|om|ozone-om|/etc/keytabs/om.keytab|om/ranger-ozone.rangernw@EXAMPLE.COM" -) - -dael_ingestor_access_url() { - echo "http://${DAEL_INGESTOR_HOST}:${DAEL_INGESTOR_HTTP_PORT}${DAEL_ACCESS_PATH}" -} - -dael_audit_event_json() { - local repo="$1" - local app_id="$2" - local evt_ms - evt_ms="$(python3 -c 'import time; print(int(time.time()*1000))')" - printf '[{"repo":"%s","reqUser":"e2e-audit-user","evtTime":%s,"access":"read","resource":"/e2e/path","result":1,"agent":"%s"}]' \ - "${repo}" "${evt_ms}" "${app_id}" -} - -dael_container_running() { - local name="$1" - docker ps --filter "name=^${name}$" --filter status=running --format '{{.Names}}' | grep -qx "${name}" -} - -dael_access_post_from_container() { - local src_container="$1" - local keytab="$2" - local principal="$3" - local service_name="$4" - local app_id="$5" - local body - local url - local outfile="/tmp/dael-access-body-${src_container}-$$" - local krb_cc="/tmp/dael-access-krb-${src_container}-$$" - local body_file="/tmp/dael-access-req-${src_container}-$$" - - body="$(dael_audit_event_json "${service_name}" "${app_id}")" - url="$(dael_ingestor_access_url)?serviceName=${service_name}&appId=${app_id}" - - if ! dael_container_running "${src_container}"; then - DAEL_ACCESS_CODE="000" - DAEL_ACCESS_BODY="container ${src_container} not running" - return 1 - fi - - if ! docker exec "${src_container}" test -f "${keytab}" 2>/dev/null; then - DAEL_ACCESS_CODE="000" - DAEL_ACCESS_BODY="keytab missing: ${keytab}" - return 1 - fi - - printf '%s' "${body}" | docker exec -i "${src_container}" tee "${body_file}" >/dev/null - DAEL_ACCESS_CODE="$(docker exec "${src_container}" bash -c \ - "export KRB5CCNAME=${krb_cc}; kinit -kt '${keytab}' '${principal}' >/dev/null 2>&1 && \ -curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X POST \ - -H 'Content-Type: application/json' -d @${body_file} '${url}'")" - DAEL_ACCESS_BODY="$(docker exec "${src_container}" cat "${outfile}" 2>/dev/null || true)" - docker exec "${src_container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true -} - -dael_expect_access() { - local label="$1" - local expect_code="$2" - if [[ "${DAEL_ACCESS_CODE}" == "${expect_code}" ]]; then - pp_record_pass "${label} (HTTP ${expect_code})" - return 0 - fi - pp_record_fail "${label} expected HTTP ${expect_code}, got ${DAEL_ACCESS_CODE}: ${DAEL_ACCESS_BODY}" - return 1 -} - -dael_json_authenticated_user() { - printf '%s' "${DAEL_ACCESS_BODY}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('authenticatedUser',''))" 2>/dev/null || echo "" -} - -dael_plan_update_services_body() { - local plan_json="$1" - local plugin_id="$2" - local repo="$3" - local users_csv="$4" - printf '%s' "${plan_json}" | python3 -c \ - "import sys,json; plan=json.load(sys.stdin); plugin_id,repo,users=sys.argv[1:4]; \ -users=[u.strip() for u in users.split(',') if u.strip()]; \ -body={'expectedVersion': plan['version'], 'updateServices': {repo: {'allowedUsers': users}}}; \ -print(json.dumps(body))" "${plugin_id}" "${repo}" "${users_csv}" -} - -dael_patch_allowed_users() { - local repo="$1" - local plugin_id="$2" - local users_csv="$3" - local patch_url - local patch_body - - patch_url="$(pp_ingestor_plan_url "${CONTAINER}")" - pp_ingestor_request "${CONTAINER}" GET "${patch_url}" - if [[ "${HTTP_CODE}" != "200" ]]; then - pp_record_fail "GET plan before PATCH allowedUsers for ${repo}: ${HTTP_CODE}" - return 1 - fi - patch_body="$(dael_plan_update_services_body "${HTTP_BODY}" "${plugin_id}" "${repo}" "${users_csv}")" - pp_ingestor_request "${CONTAINER}" PATCH "${patch_url}/plugins/${plugin_id}" "${patch_body}" - if [[ "${HTTP_CODE}" == "200" ]]; then - return 0 - fi - pp_record_fail "PATCH updateServices for ${repo} (${plugin_id}) failed: HTTP ${HTTP_CODE} ${HTTP_BODY}" - return 1 -} - -dael_onboard_repo() { - local repo="$1" - local plugin_id="$2" - local short_name="$3" - local partition_count="${4:-2}" - local version="$5" - local url body - - url="$(pp_ingestor_plan_url "${CONTAINER}")/plugins" - body="$(pp_build_onboard_plugin_json "${plugin_id}" "${partition_count}" "${version}" "${repo}:${short_name}")" - pp_ingestor_request "${CONTAINER}" POST "${url}" "${body}" - [[ "${HTTP_CODE}" == "200" ]] -} - -dael_wait_auth_to_local_applied() { - local container="$1" - local timeout="${2:-60}" - local deadline=$((SECONDS + timeout)) - while (( SECONDS < deadline )); do - if docker logs "${container}" 2>&1 | tail -200 | grep -q "Applied composed auth_to_local rules"; then - return 0 - fi - sleep 2 - done - return 1 -} - -dael_write_curl_cookbook() { - local out_file="$1" - local access_url onboard_url - access_url="$(dael_ingestor_access_url)" - onboard_url="http://${DAEL_INGESTOR_HOST}:${DAEL_INGESTOR_HTTP_PORT}${DAEL_ONBOARD_PATH}" - - { - echo "# Ranger audit ingestor access E2E — curl cookbook" - echo "# Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "# Run inside a container on rangernw with the plugin keytab (SPNEGO Host must be FQDN)." - echo "" - echo "# Health (no auth):" - echo "curl -s http://localhost:7081/api/audit/health" - echo "" - echo "# Partition plan (HTTP SPNEGO as ingestor HTTP service):" - echo "kinit -kt /etc/keytabs/HTTP.keytab HTTP/ranger-audit-ingestor.rangernw@EXAMPLE.COM" - echo "curl -s --negotiate -u : http://ranger-audit-ingestor.rangernw:7081/api/audit/partition-plan" - echo "" - echo "# Onboard plugin + allowlist (dynamic mode, services required):" - echo "curl -s --negotiate -u : -X POST -H 'Content-Type: application/json' \\" - echo " '${onboard_url}' \\" - echo " -d '{\"pluginId\":\"trino\",\"partitionCount\":3,\"expectedVersion\":1,\"services\":{\"dev_trino\":{\"allowedUsers\":[\"trino\"]}}}'" - echo "" - - local profile repo app_id short_name src_container keytab principal body - for profile in "${DAEL_PLUGIN_PROFILES[@]}"; do - IFS='|' read -r repo app_id short_name src_container keytab principal <<< "${profile}" - body="$(dael_audit_event_json "${repo}" "${app_id}")" - echo "# POST audit — ${repo} (${short_name} via auth_to_local, appId=${app_id})" - echo "# Run on: ${src_container}" - echo "kinit -kt ${keytab} ${principal}" - echo "curl -s -w '\\nHTTP %{http_code}\\n' --negotiate -u : -X POST \\" - echo " -H 'Content-Type: application/json' \\" - echo " '${access_url}?serviceName=${repo}&appId=${app_id}' \\" - echo " -d '${body}'" - echo "" - done - } > "${out_file}" - echo " Wrote curl cookbook: ${out_file}" -} - -dael_test_plugin_access() { - local profile="$1" - local repo app_id short_name src_container keytab principal auth_user - - IFS='|' read -r repo app_id short_name src_container keytab principal <<< "${profile}" - - if ! dael_container_running "${src_container}"; then - echo " SKIP ${repo}: ${src_container} not running" - return 0 - fi - - echo "" - echo " Plugin ${app_id} / repo ${repo} (shortName=${short_name})..." - dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" - if ! dael_expect_access "${repo} POST /access as ${short_name}" "200"; then - return 1 - fi - auth_user="$(dael_json_authenticated_user)" - if [[ "${auth_user}" == "${short_name}" ]]; then - pp_record_pass "${repo} authenticatedUser=${auth_user}" - else - pp_record_fail "${repo} authenticatedUser expected ${short_name}, got ${auth_user}" - return 1 - fi - return 0 -} - -dael_test_dynamic_user_toggle() { - local repo="$1" - local short_name="$2" - local src_container="$3" - local keytab="$4" - local principal="$5" - local app_id="$6" - - echo "" - echo " Dynamic allowlist toggle for ${repo}..." - - if ! dael_patch_allowed_users "${repo}" "${app_id}" "wronguser-not-allowed"; then - return 1 - fi - if dael_wait_auth_to_local_applied "${CONTAINER}" 30; then - pp_record_pass "auth_to_local recomposed after allowlist changed" - else - pp_record_fail "no auth_to_local recompose log after allowlist changed" - fi - - dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" - dael_expect_access "${repo} denied when user not in allowlist" "403" || return 1 - - if ! dael_patch_allowed_users "${repo}" "${app_id}" "${short_name}"; then - return 1 - fi - if dael_wait_auth_to_local_applied "${CONTAINER}" 30; then - pp_record_pass "auth_to_local recomposed after allowlist restored" - else - pp_record_fail "no auth_to_local recompose log after allowlist restored" - fi - - dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${repo}" "${app_id}" - dael_expect_access "${repo} allowed after allowlist restored" "200" || return 1 - return 0 -} - -dael_test_cross_repo_denied() { - local src_container="$1" - local keytab="$2" - local principal="$3" - local wrong_repo="$4" - local app_id="$5" - local label="$6" - - dael_access_post_from_container "${src_container}" "${keytab}" "${principal}" "${wrong_repo}" "${app_id}" - dael_expect_access "${label}" "403" -} - -dael_filter_profiles() { - local want="$1" - local profile - for profile in "${DAEL_PLUGIN_PROFILES[@]}"; do - IFS='|' read -r _ app_id _ _ _ _ <<< "${profile}" - if [[ -z "${want}" || ",${want}," == *",${app_id},"* || ",${want}," == *",$(echo "${profile}" | cut -d'|' -f1),"* ]]; then - echo "${profile}" - fi - done -} diff --git a/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh deleted file mode 100755 index 638f6187edd..00000000000 --- a/dev-support/ranger-docker/scripts/audit/dynamic-partition-plugin-e2e-lib.sh +++ /dev/null @@ -1,346 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# E2E helpers: dynamic partition-plan POST /plugins (mandatory services) + Kafka partition verification. -# Requires partition-plan-e2e-lib.sh and dynamic-auth-to-local-e2e-lib.sh. - -# repo|pluginId|shortName|partitionCount|source_container|keytab|principal -# partitionCount is the default for POST /plugins when not overridden. -readonly -a DPP_PLUGIN_ONBOARD_SPECS=( - "dev_hdfs|hdfs|hdfs|2|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" - "dev_hive|hiveServer2|hive|2|ranger-hive|/etc/keytabs/hive.keytab|hive/ranger-hive.rangernw@EXAMPLE.COM" - "dev_hbase|hbaseMaster|hbase|2|ranger-hbase|/etc/keytabs/hbase.keytab|hbase/ranger-hbase.rangernw@EXAMPLE.COM" - "dev_kafka|kafka|kafka|2|ranger-kafka|/etc/keytabs/kafka.keytab|kafka/ranger-kafka.rangernw@EXAMPLE.COM" - "dev_knox|knox|knox|2|ranger-knox|/etc/keytabs/knox.keytab|knox/ranger-knox.rangernw@EXAMPLE.COM" - "dev_kms|kms|rangerkms|2|ranger-kms|/etc/keytabs/rangerkms.keytab|rangerkms/ranger-kms.rangernw@EXAMPLE.COM" - "dev_ozone|ozone|om|2|ozone-om|/etc/keytabs/om.keytab|om/ranger-ozone.rangernw@EXAMPLE.COM" -) - -dpp_filter_specs() { - local want="$1" - local spec repo plugin_id - for spec in "${DPP_PLUGIN_ONBOARD_SPECS[@]}"; do - IFS='|' read -r repo plugin_id _ _ _ _ _ <<< "${spec}" - if [[ -z "${want}" || ",${want}," == *",${plugin_id},"* || ",${want}," == *",${repo},"* ]]; then - echo "${spec}" - fi - done -} - -dpp_plan_version() { - local plan_url - plan_url="$(pp_ingestor_plan_url "${CONTAINER}")" - pp_ingestor_request "${CONTAINER}" GET "${plan_url}" - if [[ "${HTTP_CODE}" != "200" ]]; then - echo "" - return 1 - fi - pp_json_field "${HTTP_BODY}" version -} - -dpp_plan_has_plugin() { - local plugin_id="$1" - local plan_json="$2" - printf '%s' "${plan_json}" | python3 -c \ - "import sys,json; d=json.load(sys.stdin); sys.exit(0 if sys.argv[1] in (d.get('plugins') or {}) else 1)" \ - "${plugin_id}" 2>/dev/null -} - -dpp_plan_plugin_partitions() { - local plugin_id="$1" - local plan_json="$2" - printf '%s' "${plan_json}" | python3 -c \ - "import sys,json; d=json.load(sys.stdin); p=(d.get('plugins') or {}).get(sys.argv[1]); \ -print(','.join(str(x) for x in (p or {}).get('partitions') or []))" "${plugin_id}" 2>/dev/null || echo "" -} - -dpp_allowed_users_for_repo() { - local repo="$1" - local short_name="$2" - if [[ "${repo}" == "dev_ozone" ]]; then - printf '%s' "om,ozone" - elif [[ "${repo}" == "dev_hdfs" ]]; then - printf '%s' "hdfs,nn" - else - printf '%s' "${short_name}" - fi -} - -dpp_plan_service_has_users() { - local repo="$1" - local users_csv="$2" - local plan_json="$3" - printf '%s' "${plan_json}" | python3 -c \ - 'import sys,json; d=json.load(sys.stdin); repo=sys.argv[1]; want=set(u.strip() for u in sys.argv[2].split(",") if u.strip()); \ -entry=(d.get("services") or {}).get(repo) or {}; have=set(entry.get("allowedUsers") or []); sys.exit(0 if want.issubset(have) else 1)' \ - "${repo}" "${users_csv}" 2>/dev/null -} - -dpp_onboard_repo_multi() { - local repo="$1" - local plugin_id="$2" - local partition_count="$3" - local users_csv="$4" - local version="$5" - local url body - - url="$(pp_ingestor_plan_url "${CONTAINER}")/plugins" - body="$(pp_build_onboard_plugin_json "${plugin_id}" "${partition_count}" "${version}" "${repo}:${users_csv}")" - pp_ingestor_request "${CONTAINER}" POST "${url}" "${body}" - [[ "${HTTP_CODE}" == "200" ]] -} - -dpp_ensure_plugin_onboarded() { - local spec="$1" - local repo plugin_id short_name part_count container _keytab _principal - local version plan_url users_csv - - IFS='|' read -r repo plugin_id short_name part_count container _keytab _principal <<< "${spec}" - - if ! dael_container_running "${container}"; then - echo " SKIP onboard ${plugin_id}: ${container} not running" - return 0 - fi - - plan_url="$(pp_ingestor_plan_url "${CONTAINER}")" - pp_ingestor_request "${CONTAINER}" GET "${plan_url}" - if [[ "${HTTP_CODE}" != "200" ]]; then - pp_record_fail "GET plan before onboard ${plugin_id}: ${HTTP_CODE}" - return 1 - fi - - users_csv="$(dpp_allowed_users_for_repo "${repo}" "${short_name}")" - - if dpp_plan_has_plugin "${plugin_id}" "${HTTP_BODY}"; then - if dpp_plan_service_has_users "${repo}" "${users_csv}" "${HTTP_BODY}"; then - pp_record_pass "${plugin_id} already in partition plan (allowlist OK)" - return 0 - fi - pp_record_fail "${plugin_id} in plan but allowlist missing [${users_csv}] — rerun with --fresh-plan" - return 1 - fi - - version="$(pp_json_field "${HTTP_BODY}" version)" - - echo " Onboard ${repo} pluginId=${plugin_id} partitions=${part_count} allowedUsers=[${users_csv}]..." - if ! dpp_onboard_repo_multi "${repo}" "${plugin_id}" "${part_count}" "${users_csv}" "${version}"; then - pp_record_fail "POST /plugins ${repo} failed: HTTP ${HTTP_CODE} ${HTTP_BODY}" - return 1 - fi - - pp_record_pass "POST /plugins ${plugin_id} -> plan v$(pp_json_field "${HTTP_BODY}" version)" - if dael_wait_auth_to_local_applied "${CONTAINER}" 45; then - pp_record_pass "auth_to_local recomposed after onboard ${plugin_id}" - else - pp_record_fail "no auth_to_local recompose log after onboard ${plugin_id}" - fi - return 0 -} - -dpp_audit_event_with_marker() { - local repo="$1" - local app_id="$2" - local marker="$3" - local evt_ms - evt_ms="$(python3 -c 'import time; print(int(time.time()*1000))')" - printf '[{"repo":"%s","reqUser":"e2e-partition","evtTime":%s,"access":"read","resource":"/e2e/%s","result":1,"agent":"%s"}]' \ - "${repo}" "${evt_ms}" "${marker}" "${app_id}" -} - -dpp_post_access_with_marker() { - local src_container="$1" - local keytab="$2" - local principal="$3" - local repo="$4" - local app_id="$5" - local marker="$6" - local body url outfile krb_cc body_file - - body="$(dpp_audit_event_with_marker "${repo}" "${app_id}" "${marker}")" - url="$(dael_ingestor_access_url)?serviceName=${repo}&appId=${app_id}" - outfile="/tmp/dpp-access-body-${src_container}-$$" - krb_cc="/tmp/dpp-access-krb-${src_container}-$$" - body_file="/tmp/dpp-access-req-${src_container}-$$" - - printf '%s' "${body}" | docker exec -i "${src_container}" tee "${body_file}" >/dev/null - DAEL_ACCESS_CODE="$(docker exec "${src_container}" bash -c \ - "export KRB5CCNAME=${krb_cc}; kinit -kt '${keytab}' '${principal}' >/dev/null 2>&1 && \ -curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X POST \ - -H 'Content-Type: application/json' -d @${body_file} '${url}'")" - DAEL_ACCESS_BODY="$(docker exec "${src_container}" cat "${outfile}" 2>/dev/null || true)" - docker exec "${src_container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true -} - -dpp_kafka_find_partition_for_marker() { - local record_key="$1" - local marker="$2" - local partitions_csv="${3:-}" - local max_messages="${4:-800}" - local out found partition_id end_offset start_offset - - if [[ -n "${partitions_csv}" ]]; then - for partition_id in $(printf '%s' "${partitions_csv}" | tr ',' ' '); do - [[ -n "${partition_id}" ]] || continue - end_offset="$(pp_kafka_run "/opt/kafka/bin/kafka-get-offsets.sh \ - --bootstrap-server ranger-kafka.rangernw:9092 --topic '${AUDIT_TOPIC}' --partitions ${partition_id} --time -1" \ - | awk -F: '{print $NF}' | tail -1)" - [[ -n "${end_offset}" && "${end_offset}" =~ ^[0-9]+$ ]] || continue - [[ "${end_offset}" -gt 0 ]] || continue - start_offset=$((end_offset > 120 ? end_offset - 120 : 0)) - out="$(pp_kafka_consumer_run "timeout 25 /opt/kafka/bin/kafka-console-consumer.sh \ - --bootstrap-server ranger-kafka.rangernw:9092 \ - --topic '${AUDIT_TOPIC}' \ - --partition ${partition_id} \ - --offset ${start_offset} \ - --max-messages 120 \ - --property print.key=true \ - --timeout-ms 15000")" - found="$(printf '%s' "${out}" | python3 -c " -import sys -key, marker, pid = sys.argv[1:4] -for line in sys.stdin: - if key in line and marker in line: - print(pid) - break -" "${record_key}" "${marker}" "${partition_id}" 2>/dev/null || true)" - if [[ -n "${found}" ]]; then - echo "${found}" - return 0 - fi - done - return 1 - fi - - out="$(pp_kafka_consumer_run "timeout 25 /opt/kafka/bin/kafka-console-consumer.sh \ - --bootstrap-server ranger-kafka.rangernw:9092 \ - --topic '${AUDIT_TOPIC}' \ - --from-beginning \ - --max-messages ${max_messages} \ - --property print.partition=true \ - --property print.key=true \ - --timeout-ms 15000")" - - printf '%s' "${out}" | python3 -c " -import re -import sys -key, marker = sys.argv[1:3] -for line in sys.stdin: - if key not in line or marker not in line: - continue - match = re.search(r'Partition[:\\s]+(\\d+)', line, re.I) - if match: - print(match.group(1)) - break -" "${record_key}" "${marker}" 2>/dev/null || true -} - -dpp_verify_plugin_partition_routing() { - local spec="$1" - local repo plugin_id short_name _part_count container keytab principal - local marker assigned part found - - IFS='|' read -r repo plugin_id short_name _part_count container keytab principal <<< "${spec}" - - if ! dael_container_running "${container}"; then - return 0 - fi - - pp_ingestor_request "${CONTAINER}" GET "$(pp_ingestor_plan_url "${CONTAINER}")" - if [[ "${HTTP_CODE}" != "200" ]]; then - pp_record_fail "GET plan for ${plugin_id} routing check" - return 1 - fi - - assigned="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" - if [[ -z "${assigned}" ]]; then - pp_record_fail "${plugin_id} not in plan — run onboard first" - return 1 - fi - pp_record_pass "${plugin_id} assigned partitions [${assigned}]" - - marker="partition-e2e-${plugin_id}-$(date +%s)" - dpp_post_access_with_marker "${container}" "${keytab}" "${principal}" "${repo}" "${plugin_id}" "${marker}" - if [[ "${DAEL_ACCESS_CODE}" != "200" && "${DAEL_ACCESS_CODE}" != "202" ]]; then - pp_record_fail "${plugin_id} POST /access for partition test: HTTP ${DAEL_ACCESS_CODE} ${DAEL_ACCESS_BODY}" - return 1 - fi - pp_record_pass "${plugin_id} POST /access HTTP ${DAEL_ACCESS_CODE}" - - found="" - for _attempt in 1 2 3 4 5; do - sleep 4 - found="$(dpp_kafka_find_partition_for_marker "${plugin_id}" "${marker}" "${assigned}")" - [[ -n "${found}" ]] && break - done - if [[ -z "${found}" ]]; then - pp_record_fail "${plugin_id} audit not found on Kafka (key=${plugin_id}, marker=${marker}) — check ACLs or dispatcher lag" - return 1 - fi - - if printf '%s' "${assigned}" | python3 -c "import sys; allowed=set(sys.argv[1].split(',')); sys.exit(0 if sys.argv[2] in allowed else 1)" "${assigned}" "${found}"; then - pp_record_pass "${plugin_id} Kafka partition ${found} in assigned [${assigned}]" - return 0 - fi - pp_record_fail "${plugin_id} Kafka partition ${found} NOT in assigned [${assigned}]" - return 1 -} - -dpp_run_harness_plugin_smoke() { - local script="$1" - local label="$2" - if [[ -x "${script}" ]]; then - echo "" - echo " Harness smoke: ${label}..." - if "${script}" 2>/dev/null; then - pp_record_pass "harness ${label} smoke" - else - pp_record_fail "harness ${label} smoke" - fi - fi -} - -dpp_optional_harness_triggers() { - local docker_dir="$1" - dpp_run_harness_plugin_smoke "${docker_dir}/scripts/kms/trigger-kms-plugin-audit-e2e.sh" "KMS" - dpp_run_harness_plugin_smoke "${docker_dir}/scripts/kafka/trigger-kafka-plugin-audit-e2e.sh" "Kafka" - dpp_run_harness_plugin_smoke "${docker_dir}/scripts/knox/trigger-knox-plugin-audit-e2e.sh" "Knox" - dpp_run_harness_plugin_smoke "${docker_dir}/scripts/hbase/trigger-hbase-plugin-audit-e2e.sh" "HBase" -} - -# Solr dispatcher Kerberos can break after ingestor/topic restarts; restart before HDFS pipeline smoke. -dpp_ensure_solr_dispatcher_ready() { - local container="ranger-audit-dispatcher-solr" - local site="/opt/ranger/audit-dispatcher/conf/ranger-audit-dispatcher-solr-site.xml" - local keytab="/etc/keytabs/rangerauditserver.keytab" - local principal="rangerauditserver/ranger-audit-dispatcher-solr.rangernw@EXAMPLE.COM" - - if ! docker ps --filter "name=^${container}$" --filter status=running -q | grep -q .; then - pp_record_fail "Solr dispatcher not running" - return 1 - fi - if docker exec "${container}" test -f "${site}" 2>/dev/null; then - pp_patch_site_prop "${container}" "${site}" "xasecure.audit.jaas.Client.option.useTicketCache" "false" || true - fi - echo " restarting ${container} (Solr Kerberos + consumer reset)..." - docker restart "${container}" >/dev/null - sleep 30 - if docker exec "${container}" bash -c "kinit -kt '${keytab}' '${principal}'" >/dev/null 2>&1 \ - && curl -sf "http://localhost:7091/api/health/ping" >/dev/null 2>&1; then - return 0 - fi - pp_record_fail "Solr dispatcher not healthy after restart" - return 1 -} diff --git a/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh deleted file mode 100755 index 4ef44502c8f..00000000000 --- a/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh +++ /dev/null @@ -1,624 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Shared helpers for partition-plan E2E scripts. - -PP_E2E_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# shellcheck source=audit-stack-lib.sh -source "${PP_E2E_LIB_DIR}/scripts/audit/audit-stack-lib.sh" - -CONTAINER="${AUDIT_INGESTOR_CONTAINER:-ranger-audit-ingestor}" -REPLICA_CONTAINER="${AUDIT_INGESTOR_REPLICA_CONTAINER:-ranger-audit-ingestor-2}" -KAFKA_CONTAINER="${KAFKA_CONTAINER:-ranger-kafka}" -PROP_DYNAMIC="ranger.audit.ingestor.kafka.partition.plan.dynamic.enabled" -PLAN_TOPIC="ranger_audit_partition_plan" -AUDIT_TOPIC="ranger_audits" -INGESTOR_PLAN_PATH="/api/audit/partition-plan" -KEYTAB="/etc/keytabs/HTTP.keytab" -HTTP_PRINCIPAL="HTTP/ranger-audit-ingestor.rangernw@EXAMPLE.COM" -INGESTOR_HTTP_PORT="7081" -KAFKA_KEYTAB="/etc/keytabs/kafka.keytab" -KAFKA_PRINCIPAL="kafka/ranger-kafka.rangernw@EXAMPLE.COM" -SITE_XMLS=( - "/opt/ranger/audit-ingestor/conf/ranger-audit-ingestor-site.xml" - "/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" -) - -# Example plugin list for E2E promote tests when site XML leaves configured.plugins empty. -PP_CONFIGURED_PLUGINS="hdfs,yarn,knox,hiveServer2,hiveMetastore,kafka,hbaseRegional,hbaseMaster,solr,trino,ozone,kudu,nifi" -PP_BUFFER_PROMOTE_CANDIDATES=(storm ambari atlas impala sqoop kylin presto elasticsearch) -PP_DEFAULT_PROMOTE_PLUGIN="storm" - -PP_PASS=0 -PP_FAIL=0 -HTTP_CODE="" -HTTP_BODY="" - -pp_record_pass() { - PP_PASS=$((PP_PASS + 1)) - echo " PASS: $*" -} - -pp_record_fail() { - PP_FAIL=$((PP_FAIL + 1)) - echo " FAIL: $*" >&2 -} - -pp_require_container() { - local name="$1" - if ! docker ps --filter "name=^${name}$" --filter status=running --format '{{.Names}}' | grep -qx "${name}"; then - echo "ERROR: container ${name} is not running" >&2 - exit 1 - fi -} - -pp_json_field() { - local json="$1" - local field="$2" - printf '%s' "${json}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('${field}',''))" 2>/dev/null || echo "" -} - -# Build POST /partition-plan/plugins JSON. services_spec: "repo1:user1,user2|repo2:user3" -pp_build_onboard_plugin_json() { - local plugin_id="$1" - local partition_count="$2" - local expected_version="$3" - local services_spec="$4" - python3 - "${plugin_id}" "${partition_count}" "${expected_version}" "${services_spec}" <<'PY' -import json, sys -plugin_id, part, ver, spec = sys.argv[1:5] -services = {} -for entry in spec.split("|"): - entry = entry.strip() - if not entry: - continue - repo, users = entry.split(":", 1) - services[repo.strip()] = {"allowedUsers": [u.strip() for u in users.split(",") if u.strip()]} -print(json.dumps({ - "pluginId": plugin_id, - "partitionCount": int(part), - "expectedVersion": int(ver), - "services": services, -})) -PY -} - -pp_read_site_prop() { - local container="$1" - local prop="$2" - local site path - for site in "${SITE_XMLS[@]}"; do - if docker exec "${container}" test -f "${site}" 2>/dev/null; then - path="${site}" - break - fi - done - [[ -n "${path:-}" ]] || return 1 - docker exec "${container}" python3 -c \ - "import xml.etree.ElementTree as ET; root=ET.parse('${path}').getroot(); want='${prop}'; \ -[print((v.text or '').strip()) for p in root.findall('property') for n in [p.find('name')] if n is not None and (n.text or '').strip()==want for v in [p.find('value')] if v is not None]" \ - 2>/dev/null || true -} - -pp_expected_topic_partitions() { - local container="$1" - local plugins per buffer count - plugins="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" - per="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions.per.configured.plugin")" - buffer="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions.buffer")" - per="${per:-3}" - buffer="${buffer:-9}" - if [[ -z "${plugins}" ]]; then - per="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.topic.partitions")" - echo "${per:-10}" - return 0 - fi - count="$(printf '%s' "${plugins}" | python3 -c "import sys; print(len([p for p in sys.stdin.read().split(',') if p.strip()]))")" - echo $(( count * per + buffer )) -} - -pp_pick_buffer_promote_plugin() { - local container="$1" - local plan_url="$2" - local configured="${3:-}" - local candidate - - if [[ -z "${configured}" ]]; then - configured="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" - fi - if [[ -z "${configured}" ]]; then - configured="${PP_CONFIGURED_PLUGINS}" - fi - - for candidate in "${PP_BUFFER_PROMOTE_CANDIDATES[@]}"; do - if [[ ",${configured}," == *",${candidate},"* ]]; then - continue - fi - pp_ingestor_request "${container}" GET "${plan_url}" - if [[ "${HTTP_CODE}" != "200" ]]; then - continue - fi - if ! printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if sys.argv[1] in (d.get('plugins') or {}) else 1)" "${candidate}" 2>/dev/null; then - echo "${candidate}" - return 0 - fi - done - # Prior test runs may have promoted all named candidates; use a unique buffer plugin id. - local synthetic="e2eBuffer$(date +%s)" - echo "${synthetic}" -} - -pp_ingestor_host() { - local container="$1" - if [[ "${container}" == "${REPLICA_CONTAINER}" ]]; then - echo "${REPLICA_CONTAINER}.rangernw" - else - echo "${CONTAINER}.rangernw" - fi -} - -# SPNEGO requires the HTTP Host to match the keytab principal (FQDN, not localhost). -pp_ingestor_plan_url() { - local container="$1" - echo "http://$(pp_ingestor_host "${container}"):${INGESTOR_HTTP_PORT}${INGESTOR_PLAN_PATH}" -} - -pp_http_principal_for() { - local container="$1" - if [[ "${container}" == "${REPLICA_CONTAINER}" ]]; then - echo "HTTP/${REPLICA_CONTAINER}.rangernw@EXAMPLE.COM" - else - echo "${HTTP_PRINCIPAL}" - fi -} - -pp_ingestor_request() { - local container="${1:?}" - local method="${2:?}" - local url="${3:?}" - local body="${4:-}" - local outfile="/tmp/pp-e2e-body-${container}-$$" - local krb_cc="/tmp/pp-e2e-krb-${container}-$$" - local body_file="" - local curl_cmd - local principal - principal="$(pp_http_principal_for "${container}")" - - curl_cmd="export KRB5CCNAME=${krb_cc}; kinit -kt ${KEYTAB} ${principal} >/dev/null 2>&1" - if [[ -n "${body}" ]]; then - body_file="/tmp/pp-e2e-req-${container}-$$" - printf '%s' "${body}" | docker exec -i "${container}" tee "${body_file}" >/dev/null - curl_cmd+="; curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X ${method}" - curl_cmd+=" -H 'Content-Type: application/json' -d @${body_file} '${url}'" - else - curl_cmd+="; curl -s -o ${outfile} -w '%{http_code}' --negotiate -u : -X ${method} '${url}'" - fi - - HTTP_CODE="$(docker exec "${container}" bash -c "${curl_cmd}")" - HTTP_BODY="$(docker exec "${container}" cat "${outfile}" 2>/dev/null || true)" - docker exec "${container}" rm -f "${outfile}" "${body_file}" "${krb_cc}" 2>/dev/null || true -} - -pp_dynamic_enabled() { - local container="$1" - local path="$2" - docker exec "${container}" grep -A1 "${PROP_DYNAMIC}" "${path}" 2>/dev/null | grep -qi 'true' -} - -pp_patch_site_prop() { - local container="$1" - local site="$2" - local prop="$3" - local value="$4" - docker exec -i -u root "${container}" python3 - "${site}" "${prop}" "${value}" <<'PY' -import sys -import xml.etree.ElementTree as ET - -path, prop, value = sys.argv[1:4] -root = ET.parse(path).getroot() -found = False -for p in root.findall("property"): - name = p.find("name") - if name is None or (name.text or "").strip() != prop: - continue - val = p.find("value") - if val is None: - val = ET.SubElement(p, "value") - val.text = value - found = True - break -if not found: - prop_el = ET.SubElement(root, "property") - ET.SubElement(prop_el, "name").text = prop - ET.SubElement(prop_el, "value").text = value -ET.indent(root, space=" ") -tree = ET.ElementTree(root) -tree.write(path, encoding="unicode", xml_declaration=False) -PY -} - -pp_ensure_audit_partitioner_for_dynamic() { - local container="${1:-${CONTAINER}}" - local seed_plugin="${2:-${PP_DYNAMIC_PRODUCER_PLUGIN:-hdfs}}" - local plugins - plugins="$(pp_read_site_prop "${container}" "ranger.audit.ingestor.kafka.configured.plugins")" - if [[ -n "${plugins}" ]]; then - return 0 - fi - for site in "${SITE_XMLS[@]}"; do - if docker exec "${container}" test -f "${site}" 2>/dev/null; then - pp_patch_site_prop "${container}" "${site}" "ranger.audit.ingestor.kafka.configured.plugins" "${seed_plugin}" - fi - done -} - -pp_set_dynamic_enabled() { - local container="$1" - local value="$2" - local restart="${3:-true}" - local changed=false - for site in "${SITE_XMLS[@]}"; do - if ! docker exec "${container}" test -f "${site}" 2>/dev/null; then - continue - fi - pp_patch_site_prop "${container}" "${site}" "${PROP_DYNAMIC}" "${value}" - changed=true - done - if [[ "${value}" == "true" ]]; then - pp_ensure_audit_partitioner_for_dynamic "${container}" - pp_prepare_audit_topic_for_greenfield "${container}" - fi - if [[ "${changed}" == "true" && "${restart}" == "true" ]]; then - echo " Restarting ${container} (dynamic=${value})..." - PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - docker restart "${container}" >/dev/null - sleep 45 - fi -} - -pp_wait_health() { - local port="${1:-7081}" - local label="${2:-Ingestor}" - local timeout="${3:-120}" - audit_stack_wait_url "http://localhost:${port}/api/audit/health" "${label}" "${timeout}" -} - -pp_wait_watcher() { - local container="$1" - local timeout="${2:-300}" - local deadline=$((SECONDS + timeout)) - local url log_args=() - - url="$(pp_ingestor_plan_url "${container}")" - if [[ -n "${PP_RESTART_SINCE:-}" ]]; then - log_args=(--since "${PP_RESTART_SINCE}") - fi - - while (( SECONDS < deadline )); do - pp_ingestor_request "${container}" GET "${url}" - if [[ "${HTTP_CODE}" == "200" ]]; then - return 0 - fi - if docker logs ${log_args+"${log_args[@]}"} "${container}" 2>&1 | grep -q "Partition plan watcher ready"; then - return 0 - fi - sleep 5 - done - return 1 -} - -pp_kafka_run() { - local script="$1" - docker exec "${KAFKA_CONTAINER}" bash -c \ - "cat > /tmp/pp-kafka-client.properties <<'EOF' -security.protocol=SASL_PLAINTEXT -sasl.kerberos.service.name=kafka -EOF -cat > /tmp/pp-kafka-client-jaas.conf </dev/null || true -} - -# kafka-console-consumer uses --consumer.config (not --command-config). -pp_kafka_consumer_run() { - local script="$1" - docker exec "${KAFKA_CONTAINER}" bash -c \ - "cat > /tmp/pp-kafka-client.properties <<'EOF' -security.protocol=SASL_PLAINTEXT -sasl.kerberos.service.name=kafka -EOF -cat > /tmp/pp-kafka-client-jaas.conf </dev/null || true -} - - -pp_stop_audit_stack_consumers() { - docker stop ranger-audit-ingestor ranger-audit-dispatcher-solr ranger-audit-dispatcher-hdfs 2>/dev/null || true -} - -pp_start_audit_stack_consumers() { - docker start ranger-audit-ingestor ranger-audit-dispatcher-solr ranger-audit-dispatcher-hdfs 2>/dev/null || true -} - -pp_kafka_wait_topic_absent() { - local topic="$1" - local timeout="${2:-180}" - local deadline=$((SECONDS + timeout)) - while (( SECONDS < deadline )); do - if ! pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${topic}"; then - return 0 - fi - sleep 5 - done - echo "ERROR: Kafka topic '${topic}' still present after ${timeout}s" >&2 - return 1 -} - -pp_kafka_topic_partition_count() { - local topic="$1" - local out - out="$(pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --describe --topic '${topic}'")" - printf '%s' "${out}" | python3 -c "import sys,re; t=sys.stdin.read(); m=re.search(r'PartitionCount:\\s*(\\d+)', t); print(m.group(1) if m else '')" 2>/dev/null || echo "" -} - -pp_delete_plan_topic() { - pp_stop_audit_stack_consumers - pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${PLAN_TOPIC}'" - pp_kafka_wait_topic_absent "${PLAN_TOPIC}" 180 || true - pp_start_audit_stack_consumers - sleep 10 -} - -# Greenfield bootstrap requires plan.topicPartitionCount == Kafka ranger_audits partitions. -pp_prepare_audit_topic_for_greenfield() { - local container="${1:-${CONTAINER}}" - local expected actual - - expected="$(pp_expected_topic_partitions "${container}")" - actual="$(pp_kafka_topic_partition_count "${AUDIT_TOPIC}")" - if [[ -z "${actual}" || "${actual}" == "${expected}" ]]; then - return 0 - fi - echo " Recreating ${AUDIT_TOPIC} (${actual} partitions -> ${expected} for site XML layout)..." - pp_stop_audit_stack_consumers - pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${AUDIT_TOPIC}'" - pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --delete --topic '${PLAN_TOPIC}'" || true - pp_kafka_wait_topic_absent "${AUDIT_TOPIC}" 180 - pp_kafka_wait_topic_absent "${PLAN_TOPIC}" 180 || true - pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --create --topic '${AUDIT_TOPIC}' --partitions ${expected} --replication-factor 1" - sleep 3 - pp_start_audit_stack_consumers - sleep 10 -} - -pp_seed_plan_json() { - local json="$1" - local tmp - tmp="$(mktemp)" - printf '%s' "${json}" > "${tmp}" - pp_seed_plan_file "${tmp}" - rm -f "${tmp}" -} - -pp_seed_plan_file() { - local file="$1" - pp_ensure_plan_topic - { printf '%s:' "${AUDIT_TOPIC}"; cat "${file}"; } | docker exec -i "${KAFKA_CONTAINER}" bash -c \ - "cat > /tmp/pp-kafka-client.properties <<'EOF' -security.protocol=SASL_PLAINTEXT -sasl.kerberos.service.name=kafka -EOF -cat > /tmp/pp-kafka-client-jaas.conf </dev/null 2>&1 || true - sleep 2 -} - -pp_ensure_plan_topic() { - if ! pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${PLAN_TOPIC}"; then - pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --create --topic '${PLAN_TOPIC}' --partitions 1 --replication-factor 1 --config cleanup.policy=compact" - fi -} - -pp_ensure_replica_keytab() { - local rep_fqdn="${REPLICA_CONTAINER}.rangernw" - local host_kt="${PP_E2E_LIB_DIR}/dist/keytabs/${REPLICA_CONTAINER}" - docker exec ranger-kdc bash -c " - set -euo pipefail - keytab_dir=/etc/keytabs/${REPLICA_CONTAINER} - mkdir -p \"\${keytab_dir}\" - kadmin.local -q 'addprinc -randkey HTTP/${rep_fqdn}' >/dev/null 2>&1 || true - kadmin.local -q 'addprinc -randkey rangerauditserver/${rep_fqdn}' >/dev/null 2>&1 || true - rm -f \"\${keytab_dir}/HTTP.keytab\" \"\${keytab_dir}/rangerauditserver.keytab\" - kadmin.local -q 'ktadd -k '\${keytab_dir}'/HTTP.keytab HTTP/${rep_fqdn}' - kadmin.local -q 'ktadd -k '\${keytab_dir}'/rangerauditserver.keytab rangerauditserver/${rep_fqdn}' - chmod 444 \"\${keytab_dir}/\"*.keytab - " - mkdir -p "${host_kt}" - docker cp "ranger-kdc:/etc/keytabs/${REPLICA_CONTAINER}/." "${host_kt}/" >/dev/null -} - -pp_patch_replica_site_file() { - local file="$1" - local rep_fqdn="${REPLICA_CONTAINER}.rangernw" - local primary_fqdn="${CONTAINER}.rangernw" - python3 - "${file}" "${primary_fqdn}" "${rep_fqdn}" <<'PY' -import sys -import xml.etree.ElementTree as ET - -path, primary_fqdn, rep_fqdn = sys.argv[1:4] -props = { - "ranger.audit.ingestor.host": rep_fqdn, - "ranger.audit.ingestor.bind.address": rep_fqdn, -} -root = ET.parse(path).getroot() -for prop in root.findall("property"): - name = prop.find("name") - if name is None: - continue - key = (name.text or "").strip() - if key in props: - value = prop.find("value") - if value is None: - value = ET.SubElement(prop, "value") - value.text = props[key] -ET.indent(root, space=" ") -tree = ET.ElementTree(root) -tree.write(path, encoding="unicode", xml_declaration=False) -PY -} - -pp_replica_config_staging() { - echo "${PP_E2E_LIB_DIR}/dist/${REPLICA_CONTAINER}-conf" -} - -pp_sync_replica_config_from_primary() { - local preserve_mount="${1:-false}" - local staging - staging="$(pp_replica_config_staging)" - if [[ "${preserve_mount}" == "true" ]]; then - mkdir -p "${staging}/conf" "${staging}/webapp-conf" - else - rm -rf "${staging}" - mkdir -p "${staging}/conf" "${staging}/webapp-conf" - fi - docker cp "${CONTAINER}:/opt/ranger/audit-ingestor/conf/." "${staging}/conf/" >/dev/null - if docker exec "${CONTAINER}" test -f "${SITE_XMLS[1]}" 2>/dev/null; then - docker cp "${CONTAINER}:${SITE_XMLS[1]}" "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" >/dev/null - else - cp "${staging}/conf/ranger-audit-ingestor-site.xml" "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" - fi - pp_patch_replica_site_file "${staging}/conf/ranger-audit-ingestor-site.xml" - pp_patch_replica_site_file "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" -} - -pp_start_ingestor_replica() { - local image="${1:-ranger-audit-ingestor:latest}" - local staging - if docker ps -a --format '{{.Names}}' | grep -qx "${REPLICA_CONTAINER}"; then - docker rm -f "${REPLICA_CONTAINER}" >/dev/null 2>&1 || true - fi - pp_ensure_replica_keytab - pp_sync_replica_config_from_primary - staging="$(pp_replica_config_staging)" - if ! docker run -d --name "${REPLICA_CONTAINER}" \ - --network rangernw \ - --hostname "${REPLICA_CONTAINER}.rangernw" \ - -p 7082:7081 \ - -e JAVA_HOME=/opt/java/openjdk \ - -e AUDIT_INGESTOR_HOME_DIR=/opt/ranger/audit-ingestor \ - -e AUDIT_INGESTOR_CONF_DIR=/opt/ranger/audit-ingestor/conf \ - -e AUDIT_INGESTOR_LOG_DIR=/var/log/ranger/audit-ingestor \ - -e AUDIT_INGESTOR_HEAP="-Xms512m -Xmx2g" \ - -e KERBEROS_ENABLED=true \ - -e KAFKA_BOOTSTRAP_SERVERS=ranger-kafka.rangernw:9092 \ - -v "${PP_E2E_LIB_DIR}/dist/keytabs/${REPLICA_CONTAINER}:/etc/keytabs" \ - -v "${staging}/conf:/opt/ranger/audit-ingestor/conf:ro" \ - "${image}" >/dev/null; then - echo "ERROR: docker run failed for ${REPLICA_CONTAINER}" >&2 - return 1 - fi - PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - sleep 60 - if ! pp_wait_health 7082 "Ingestor replica" 300; then - return 1 - fi - pp_install_replica_webapp_site -} - -pp_install_replica_webapp_site() { - local staging - staging="$(pp_replica_config_staging)" - if ! docker exec "${REPLICA_CONTAINER}" test -d \ - /opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf 2>/dev/null; then - return 0 - fi - docker cp "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" \ - "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" >/dev/null - PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - docker restart "${REPLICA_CONTAINER}" >/dev/null - sleep 60 - pp_wait_health 7082 "Ingestor replica after webapp site sync" 300 -} - -pp_stop_ingestor_replica() { - docker rm -f "${REPLICA_CONTAINER}" >/dev/null 2>&1 || true -} - -pp_copy_site_to_replica() { - local staging restart="${2:-true}" - pp_sync_replica_config_from_primary "true" - staging="$(pp_replica_config_staging)" - docker cp "${staging}/conf/ranger-audit-ingestor-site.xml" \ - "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/conf/ranger-audit-ingestor-site.xml" - if docker exec "${REPLICA_CONTAINER}" test -d \ - /opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf 2>/dev/null; then - docker cp "${staging}/webapp-conf/ranger-audit-ingestor-site.xml" \ - "${REPLICA_CONTAINER}:/opt/ranger/audit-ingestor/webapp/audit-ingestor/WEB-INF/classes/conf/ranger-audit-ingestor-site.xml" - fi - if [[ "${restart}" == "true" ]]; then - PP_RESTART_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - docker restart "${REPLICA_CONTAINER}" >/dev/null - sleep 45 - pp_wait_health 7082 "Ingestor replica after config copy" 300 - fi -} - -pp_print_results() { - echo "" - echo "Results: ${PP_PASS} passed, ${PP_FAIL} failed" - if [[ "${PP_FAIL}" -ne 0 ]]; then - echo "See: audit-server/README-KAFKA-PARTITION-PLAN-E2E-TEST-PLAN.md" >&2 - return 1 - fi - return 0 -} - -pp_preflight_tier3() { - chmod +x "${PP_E2E_LIB_DIR}/scripts/audit/wait-for-audit-health.sh" 2>/dev/null || true - "${PP_E2E_LIB_DIR}/scripts/audit/wait-for-audit-health.sh" --tier 3 --timeout "${1:-120}" - pp_require_container "${CONTAINER}" - pp_require_container "${KAFKA_CONTAINER}" -} diff --git a/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh deleted file mode 100755 index f3bc77ee382..00000000000 --- a/dev-support/ranger-docker/scripts/audit/verify-dynamic-partition-plugin-e2e.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# E2E: dynamic partition allocation — POST /partition-plan/plugins per plugin + Kafka routing. -# -# Flow (mirrors ranger-audit-e2e-harness plugin matrix): -# 1. Enable dynamic mode (empty configured.plugins → buffer-only bootstrap) -# 2. For each running plugin container: POST /partition-plan/plugins -# (pluginId, partitionCount, expectedVersion, mandatory services map) -# 3. POST /api/audit/access from plugin keytab → verify Kafka record partition ∈ plan -# 4. Optional: run harness trigger-* scripts when present (KMS, Kafka, Knox, HBase) -# -# Prerequisites: Tier 3 Docker (see audit-server/README-DYNAMIC-PARTITION-PLUGIN-E2E.md). -# Reference harness branch: ranger-audit-e2e-harness (plugin verify/trigger scripts). -# -# Usage: -# cd dev-support/ranger-docker -# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh -# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh --no-enable --plugins hdfs,kms -# ./scripts/audit/verify-dynamic-partition-plugin-e2e.sh --with-harness-triggers - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "${SCRIPT_DIR}" - -# shellcheck source=partition-plan-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" -# shellcheck source=dynamic-auth-to-local-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/dynamic-auth-to-local-e2e-lib.sh" -# shellcheck source=dynamic-partition-plugin-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/dynamic-partition-plugin-e2e-lib.sh" - -DO_ENABLE=true -PLUGIN_FILTER="" -WITH_HARNESS=false -TIMEOUT=300 -FRESH_PLAN=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --no-enable) DO_ENABLE=false; shift ;; - --plugins) PLUGIN_FILTER="${2:?}"; shift 2 ;; - --with-harness-triggers) WITH_HARNESS=true; shift ;; - --fresh-plan) FRESH_PLAN=true; shift ;; - --timeout) TIMEOUT="${2:?}"; shift 2 ;; - -h|--help) - sed -n '31,36p' "$0" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -echo "=== Dynamic partition plugin onboard + routing E2E ===" -pp_preflight_tier3 "${TIMEOUT}" - -if [[ "${DO_ENABLE}" == "true" ]]; then - if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then - echo "Preparing audit topic for greenfield layout..." - pp_prepare_audit_topic_for_greenfield "${CONTAINER}" - if [[ "${FRESH_PLAN}" == "true" ]]; then - pp_delete_plan_topic - fi - echo "Enabling dynamic partition plan..." - pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; pp_print_results; exit 1; } - pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health"; pp_print_results; exit 1; } - fi -fi - -if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then - pp_record_fail "dynamic.enabled must be true" - pp_print_results - exit 1 -fi - -if ! pp_wait_watcher "${CONTAINER}" "${TIMEOUT}"; then - pp_record_fail "PartitionPlanWatcher not ready" - pp_print_results - exit 1 -fi -pp_record_pass "PartitionPlanWatcher ready" - -PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" -pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" -if [[ "${HTTP_CODE}" == "200" ]]; then - pp_record_pass "GET partition-plan returns 200 (v$(pp_json_field "${HTTP_BODY}" version))" -else - pp_record_fail "GET partition-plan expected 200, got ${HTTP_CODE}" - pp_print_results - exit 1 -fi - -echo "" -echo "=== Phase 1: POST /plugins per running plugin (REST, mandatory services) ===" -spec="" -while IFS= read -r spec; do - [[ -n "${spec}" ]] || continue - dpp_ensure_plugin_onboarded "${spec}" || true -done < <(dpp_filter_specs "${PLUGIN_FILTER}") - -echo "" -echo "=== Phase 2: POST /access + Kafka partition ∈ plan ===" -while IFS= read -r spec; do - [[ -n "${spec}" ]] || continue - IFS='|' read -r repo plugin_id _ _ container _ _ <<< "${spec}" - echo "" - echo "--- ${plugin_id} (${repo}) ---" - dpp_verify_plugin_partition_routing "${spec}" || true -done < <(dpp_filter_specs "${PLUGIN_FILTER}") - -if [[ "${WITH_HARNESS}" == "true" ]]; then - echo "" - echo "=== Phase 3: optional ranger-audit-e2e-harness plugin triggers ===" - dpp_optional_harness_triggers "${SCRIPT_DIR}" -fi - -echo "" -echo "See: audit-server/README-DYNAMIC-PARTITION-PLUGIN-E2E.md" -echo "Curl cookbook: dist/audit-e2e/access-ingestor-curl-cookbook.sh (run verify-dynamic-auth-to-local-e2e.sh --generate-curl-only)" - -pp_print_results diff --git a/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh deleted file mode 100755 index a15742abe91..00000000000 --- a/dev-support/ranger-docker/scripts/audit/verify-hdfs-dynamic-partition-e2e.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# HDFS-focused dynamic partition E2E: -# 1. Enable dynamic mode (greenfield buffer-only plan) -# 2. POST /partition-plan/plugins — dev_hdfs + hdfs plugin + mandatory services allowlist -# 3. Verify POST /access + Kafka partition routing -# 4. PATCH /partition-plan/plugins/hdfs — expand partition count -# 5. Verify routing again after scale -# -# Usage: -# cd dev-support/ranger-docker -# ./scripts/audit/verify-hdfs-dynamic-partition-e2e.sh -# ./scripts/audit/verify-hdfs-dynamic-partition-e2e.sh --with-hdfs-trigger - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "${SCRIPT_DIR}" - -# shellcheck source=partition-plan-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" -# shellcheck source=dynamic-auth-to-local-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/dynamic-auth-to-local-e2e-lib.sh" -# shellcheck source=dynamic-partition-plugin-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/dynamic-partition-plugin-e2e-lib.sh" - -readonly HDFS_SPEC="dev_hdfs|hdfs|hdfs|2|ranger-hadoop|/etc/keytabs/hdfs.keytab|hdfs/ranger-hadoop.rangernw@EXAMPLE.COM" -SCALE_ADDITIONAL="${SCALE_ADDITIONAL:-2}" -TIMEOUT=300 -FRESH_PLAN=false -WITH_HDFS_TRIGGER=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --fresh-plan) FRESH_PLAN=true; shift ;; - --with-hdfs-trigger) WITH_HDFS_TRIGGER=true; shift ;; - --scale-additional) SCALE_ADDITIONAL="${2:?}"; shift 2 ;; - --timeout) TIMEOUT="${2:?}"; shift 2 ;; - -h|--help) - sed -n '26,29p' "$0" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -hdfs_scale_plugin() { - local plugin_id="$1" - local additional="$2" - local version="$3" - local url body before after attempt - - pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" - before="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" - - url="${PLAN_URL}/plugins/${plugin_id}" - body="{\"additionalPartitions\":${additional},\"expectedVersion\":${version}}" - for attempt in 1 2 3; do - echo " PATCH ${url} (+${additional} partitions, expectedVersion=${version}, attempt ${attempt})..." - pp_ingestor_request "${CONTAINER}" PATCH "${url}" "${body}" - if [[ "${HTTP_CODE}" == "200" ]]; then - after="$(dpp_plan_plugin_partitions "${plugin_id}" "${HTTP_BODY}")" - pp_record_pass "scale ${plugin_id} partitions [${before}] -> [${after}] plan v$(pp_json_field "${HTTP_BODY}" version)" - if [[ "$(echo "${before}" | tr ',' '\n' | wc -l | tr -d ' ')" -ge "$(echo "${after}" | tr ',' '\n' | wc -l | tr -d ' ')" ]]; then - pp_record_fail "scale ${plugin_id} did not increase partition count" - return 1 - fi - return 0 - fi - if [[ "${HTTP_CODE}" == "503" && "${attempt}" -lt 3 ]]; then - echo " scale returned 503 (Kafka grow timeout?) — retrying in 15s..." - sleep 15 - version="$(dpp_plan_version)" - body="{\"additionalPartitions\":${additional},\"expectedVersion\":${version}}" - continue - fi - pp_record_fail "scale ${plugin_id} expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" - return 1 - done - return 1 -} - -echo "=== HDFS dynamic partition REST + routing E2E ===" -if [[ -x "${SCRIPT_DIR}/scripts/audit/ensure-audit-ingestor-plugin-users.sh" ]]; then - "${SCRIPT_DIR}/scripts/audit/ensure-audit-ingestor-plugin-users.sh" || true -fi -for site in "${SITE_XMLS[@]}"; do - if docker exec "${CONTAINER}" test -f "${site}" 2>/dev/null; then - pp_patch_site_prop "${CONTAINER}" "${site}" "ranger.audit.ingestor.service.dev_hdfs.allowed.users" "hdfs,nn" - fi -done -pp_preflight_tier3 "${TIMEOUT}" - -if ! dael_container_running "ranger-hadoop"; then - echo "ERROR: ranger-hadoop is not running. Start HDFS stack first, e.g.:" >&2 - echo " ./setup-audit-e2e.sh up --hdfs-only --no-verify" >&2 - exit 1 -fi - -PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" - -echo "" -echo "=== Step 1: Enable dynamic partition plan ===" -if [[ "${FRESH_PLAN}" == "true" ]]; then - echo " Fresh plan requested — resetting plan topic and greenfield audit topic layout..." - pp_set_dynamic_enabled "${CONTAINER}" "false" "false" || true - pp_prepare_audit_topic_for_greenfield "${CONTAINER}" - pp_delete_plan_topic -fi -if ! pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then - pp_prepare_audit_topic_for_greenfield "${CONTAINER}" - if [[ "${FRESH_PLAN}" == "true" ]]; then - pp_delete_plan_topic - fi - pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; pp_print_results; exit 1; } - pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health"; pp_print_results; exit 1; } -else - pp_record_pass "dynamic mode already enabled" -fi - -pp_wait_watcher "${CONTAINER}" "${TIMEOUT}" || { pp_record_fail "watcher"; pp_print_results; exit 1; } -pp_record_pass "PartitionPlanWatcher ready" - -echo "" -echo "=== Step 2: POST /partition-plan/plugins (dev_hdfs + allowlist) ===" -dpp_ensure_plugin_onboarded "${HDFS_SPEC}" || { pp_print_results; exit 1; } - -echo "" -echo "=== Step 3: E2E after onboard — POST /access + Kafka partition routing ===" -dpp_verify_plugin_partition_routing "${HDFS_SPEC}" || true - -if [[ "${WITH_HDFS_TRIGGER}" == "true" && -x "${SCRIPT_DIR}/setup-audit-e2e.sh" ]]; then - echo "" - echo "=== Step 3b: Full HDFS audit pipeline (dfs smoke) ===" - dpp_ensure_solr_dispatcher_ready || true - if "${SCRIPT_DIR}/setup-audit-e2e.sh" trigger-hdfs-audit --no-verify 2>/dev/null; then - pp_record_pass "trigger-hdfs-audit pipeline" - else - pp_record_fail "trigger-hdfs-audit pipeline" - fi -fi - -echo "" -echo "=== Step 4: PATCH /partition-plan/plugins/hdfs (expand partitions) ===" -version="$(dpp_plan_version)" -if [[ -z "${version}" ]]; then - pp_record_fail "could not read plan version before scale" - pp_print_results - exit 1 -fi -hdfs_scale_plugin "hdfs" "${SCALE_ADDITIONAL}" "${version}" || true -dael_wait_auth_to_local_applied "${CONTAINER}" 45 && pp_record_pass "auth_to_local after scale" || pp_record_fail "auth_to_local after scale" -echo " waiting 10s for ingestor partitioner metadata after scale..." -sleep 10 - -echo "" -echo "=== Step 5: E2E after scale — POST /access + Kafka partition routing ===" -dpp_verify_plugin_partition_routing "${HDFS_SPEC}" || true - -if [[ "${WITH_HDFS_TRIGGER}" == "true" && -x "${SCRIPT_DIR}/setup-audit-e2e.sh" ]]; then - echo "" - echo "=== Step 5b: Full HDFS audit pipeline after scale ===" - dpp_ensure_solr_dispatcher_ready || true - if "${SCRIPT_DIR}/setup-audit-e2e.sh" trigger-hdfs-audit --no-verify 2>/dev/null; then - pp_record_pass "trigger-hdfs-audit after scale" - else - pp_record_fail "trigger-hdfs-audit after scale" - fi -fi - -pp_print_results -[[ "${PP_FAIL}" -eq 0 ]] diff --git a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh deleted file mode 100755 index b5c6911d180..00000000000 --- a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Core E2E: static regression + dynamic REST (promote/scale/409/400). -# -# Usage: -# ./scripts/audit/verify-partition-plan-e2e.sh --static-only -# ./scripts/audit/verify-partition-plan-e2e.sh --dynamic --restore-static -# ./scripts/audit/verify-partition-plan-e2e.sh --dynamic --with-audit-smoke -# -# Full suite: ./scripts/audit/verify-partition-plan-e2e-all.sh - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "${SCRIPT_DIR}" - -# shellcheck source=partition-plan-e2e-lib.sh -source "${SCRIPT_DIR}/scripts/audit/partition-plan-e2e-lib.sh" - -RUN_STATIC=false -RUN_DYNAMIC=false -DO_ENABLE=true -RESTORE_STATIC=false -AUDIT_SMOKE=false -TIMEOUT=300 -PROMOTE_PLUGIN="storm" - -while [[ $# -gt 0 ]]; do - case "$1" in - --static-only) RUN_STATIC=true; shift ;; - --dynamic) RUN_DYNAMIC=true; shift ;; - --no-enable) DO_ENABLE=false; shift ;; - --restore-static) RESTORE_STATIC=true; shift ;; - --with-audit-smoke) AUDIT_SMOKE=true; shift ;; - --timeout) TIMEOUT="${2:?}"; shift 2 ;; - --promote-plugin) PROMOTE_PLUGIN="${2:?}"; shift 2 ;; - -h|--help) - sed -n '19,25p' "$0" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -if [[ "${RUN_STATIC}" != "true" && "${RUN_DYNAMIC}" != "true" ]]; then - RUN_STATIC=true - RUN_DYNAMIC=true -fi - -PLAN_URL="$(pp_ingestor_plan_url "${CONTAINER}")" - -run_static_tests() { - echo "" - echo "=== Static mode (dynamic.enabled=false) ===" - if pp_dynamic_enabled "${CONTAINER}" "${SITE_XMLS[0]}"; then - pp_record_fail "dynamic.enabled should be false (use --restore-static)" - return - fi - pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" - if [[ "${HTTP_CODE}" == "503" ]]; then - pp_record_pass "GET partition-plan returns 503 when disabled" - else - pp_record_fail "GET expected 503, got ${HTTP_CODE}" - fi - if curl -sf "http://localhost:7081/api/audit/health" >/dev/null; then - pp_record_pass "GET /health still 200" - else - pp_record_fail "GET /health not reachable" - fi -} - -run_dynamic_tests() { - echo "" - echo "=== Dynamic mode ===" - if [[ "${DO_ENABLE}" == "true" ]]; then - echo "Preparing greenfield Kafka layout (static mode)..." - pp_set_dynamic_enabled "${CONTAINER}" "false" "false" || true - pp_prepare_audit_topic_for_greenfield "${CONTAINER}" - pp_delete_plan_topic - echo "Enabling dynamic partition plan..." - pp_set_dynamic_enabled "${CONTAINER}" "true" || { pp_record_fail "enable dynamic"; return; } - pp_wait_health 7081 "Ingestor after enable" "${TIMEOUT}" || { pp_record_fail "health after enable"; return; } - fi - if ! pp_wait_watcher "${CONTAINER}" "${TIMEOUT}"; then - pp_record_fail "PartitionPlanWatcher not ready" - return - fi - pp_record_pass "PartitionPlanWatcher ready" - - pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" - if [[ "${HTTP_CODE}" != "200" ]]; then - pp_record_fail "GET expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" - return - fi - pp_record_pass "GET partition-plan returns 200" - - local version topic_count kafka_parts new_version plan_plugins expected_plugin_count promote_plugin - version="$(pp_json_field "${HTTP_BODY}" version)" - if [[ -n "${version}" && "${version}" -ge 1 ]]; then - pp_record_pass "plan version=${version}" - else - pp_record_fail "invalid version: ${version}" - fi - - plan_plugins="$(printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('plugins') or {}))" 2>/dev/null || echo "")" - local configured_plugins - configured_plugins="$(pp_read_site_prop "${CONTAINER}" "ranger.audit.ingestor.kafka.configured.plugins")" - if [[ -n "${configured_plugins}" ]]; then - expected_plugin_count="$(printf '%s' "${configured_plugins}" | python3 -c "import sys; print(len([p for p in sys.stdin.read().split(',') if p.strip()]))")" - else - expected_plugin_count="0" - fi - topic_count="$(pp_json_field "${HTTP_BODY}" topicPartitionCount)" - kafka_parts="$(pp_kafka_topic_partition_count "${AUDIT_TOPIC}")" - if [[ "${version}" == "1" && "${plan_plugins}" == "${expected_plugin_count}" ]]; then - if [[ "${expected_plugin_count}" == "0" ]]; then - pp_record_pass "greenfield plan is buffer-only (configured.plugins empty in site XML)" - else - pp_record_pass "greenfield plan lists ${plan_plugins} configured plugins (site XML layout)" - fi - elif [[ "${version}" != "1" ]]; then - pp_record_pass "plan has ${plan_plugins} plugins at v${version}" - else - pp_record_fail "greenfield expected ${expected_plugin_count} configured plugins, plan has ${plan_plugins}" - fi - if [[ -n "${topic_count}" && "${topic_count}" == "${kafka_parts}" ]]; then - pp_record_pass "topicPartitionCount matches Kafka (${topic_count})" - else - pp_record_fail "topicPartitionCount=${topic_count} kafka=${kafka_parts}" - fi - - local services_count - services_count="$(printf '%s' "${HTTP_BODY}" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('services') or {}))" 2>/dev/null || echo "0")" - if [[ "${services_count}" -ge 1 ]]; then - pp_record_pass "plan includes services allowlist (${services_count} repos)" - else - pp_record_fail "plan missing services allowlist map" - fi - - promote_plugin="${PROMOTE_PLUGIN}" - if [[ "${PROMOTE_PLUGIN}" == "storm" ]]; then - promote_plugin="$(pp_pick_buffer_promote_plugin "${CONTAINER}" "${PLAN_URL}")" - fi - - if pp_kafka_run "/opt/kafka/bin/kafka-topics.sh --bootstrap-server ranger-kafka.rangernw:9092 --list" | grep -qx "${PLAN_TOPIC}"; then - pp_record_pass "plan topic ${PLAN_TOPIC} exists" - else - pp_record_fail "plan topic missing" - fi - - echo "" - echo " Onboard without services (400)..." - pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ - "{\"pluginId\":\"e2eNoServices\",\"partitionCount\":2,\"expectedVersion\":${version}}" - if [[ "${HTTP_CODE}" == "400" ]] && echo "${HTTP_BODY}" | grep -qi "services"; then - pp_record_pass "onboard without services -> 400" - else - pp_record_fail "onboard without services expected 400, got ${HTTP_CODE}: ${HTTP_BODY}" - fi - - echo "" - echo " Onboard ${promote_plugin} (buffer plugin + mandatory services)..." - local onboard_body onboard_repo="dev_${promote_plugin}" - onboard_body="$(pp_build_onboard_plugin_json "${promote_plugin}" 2 "${version}" "${onboard_repo}:${promote_plugin}")" - pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" "${onboard_body}" - if [[ "${HTTP_CODE}" == "200" ]]; then - new_version="$(pp_json_field "${HTTP_BODY}" version)" - if [[ "${new_version}" -gt "${version}" ]] && echo "${HTTP_BODY}" | grep -q "\"${promote_plugin}\""; then - pp_record_pass "onboard ${promote_plugin} -> v${new_version}" - version="${new_version}" - else - pp_record_fail "onboard response invalid" - fi - else - pp_record_fail "onboard expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" - fi - - echo " Stale expectedVersion (409)..." - pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ - "$(pp_build_onboard_plugin_json "e2eStale409" 2 1 "dev_e2eStale409:e2eStale409")" - if [[ "${HTTP_CODE}" == "409" ]]; then - pp_record_pass "stale expectedVersion -> 409" - else - pp_record_fail "stale expectedVersion expected 409, got ${HTTP_CODE}" - fi - - echo " Onboard hdfs again (400)..." - pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" \ - "$(pp_build_onboard_plugin_json "hdfs" 2 "${version}" "dev_hdfs:hdfs")" - if [[ "${HTTP_CODE}" == "400" ]]; then - pp_record_pass "duplicate hdfs promote -> 400" - else - pp_record_fail "duplicate promote expected 400, got ${HTTP_CODE}" - fi - - echo " Multi-service onboard (trino + dev_trino, dev_trino2)..." - local multi_body - multi_body="$(pp_build_onboard_plugin_json "trino" 2 "${version}" "dev_trino:trino|dev_trino2:trino2")" - pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/plugins" "${multi_body}" - if [[ "${HTTP_CODE}" == "200" ]]; then - new_version="$(pp_json_field "${HTTP_BODY}" version)" - if [[ "${new_version}" -gt "${version}" ]] \ - && echo "${HTTP_BODY}" | grep -q '"dev_trino"' \ - && echo "${HTTP_BODY}" | grep -q '"dev_trino2"'; then - pp_record_pass "multi-service onboard trino -> v${new_version}" - version="${new_version}" - else - pp_record_fail "multi-service onboard response invalid" - fi - else - pp_record_fail "multi-service onboard expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" - fi - - echo " Scale ${promote_plugin}..." - pp_ingestor_request "${CONTAINER}" PATCH "${PLAN_URL}/plugins/${promote_plugin}" \ - "{\"additionalPartitions\":1,\"expectedVersion\":${version}}" - if [[ "${HTTP_CODE}" == "200" ]]; then - new_version="$(pp_json_field "${HTTP_BODY}" version)" - if [[ "${new_version}" -gt "${version}" ]]; then - pp_record_pass "scale -> v${new_version}" - version="${new_version}" - else - pp_record_fail "scale did not bump version" - fi - else - pp_record_fail "scale expected 200, got ${HTTP_CODE}: ${HTTP_BODY}" - fi - - sleep 5 - pp_ingestor_request "${CONTAINER}" GET "${PLAN_URL}" - if [[ "${HTTP_CODE}" == "200" && "$(pp_json_field "${HTTP_BODY}" version)" == "${version}" ]]; then - pp_record_pass "GET stable at v${version}" - else - pp_record_fail "GET version mismatch after mutations" - fi - - if [[ "${AUDIT_SMOKE}" == "true" ]]; then - echo " Audit pipeline smoke..." - if ./scripts/audit/verify-audit-tier3-e2e.sh --timeout "${TIMEOUT}"; then - pp_record_pass "verify-audit-tier3-e2e after mutations" - else - pp_record_fail "verify-audit-tier3-e2e failed" - fi - fi -} - -echo "Partition plan E2E (static=${RUN_STATIC}, dynamic=${RUN_DYNAMIC})" -pp_preflight_tier3 "${TIMEOUT}" - -if [[ "${RUN_STATIC}" == "true" && "${RUN_DYNAMIC}" == "true" && "${DO_ENABLE}" == "true" ]]; then - run_static_tests - run_dynamic_tests -elif [[ "${RUN_STATIC}" == "true" ]]; then - run_static_tests -elif [[ "${RUN_DYNAMIC}" == "true" ]]; then - run_dynamic_tests -fi - -if [[ "${RESTORE_STATIC}" == "true" ]]; then - echo "" - echo "Restoring static mode..." - pp_set_dynamic_enabled "${CONTAINER}" "false" || true -fi - -pp_print_results From 8aeb2243dfa8eee93e266ba8ce631e77cdccf56d Mon Sep 17 00:00:00 2001 From: ramk Date: Wed, 24 Jun 2026 14:07:27 +0530 Subject: [PATCH 08/10] RANGER-5655: Remove legacy partition-plan models and cache admin config Drop unused PromotePlugin, OnboardService, PluginScale, and PartitionPlanReplacement after REST API consolidation. Cache partition-plan admin users and dynamic.enabled flag in PartitionPlanService constructor. --- .../kafka/partition/PartitionPlanService.java | 25 ++- .../kafka/partition/model/OnboardService.java | 69 ------- .../model/PartitionPlanReplacement.java | 195 ------------------ .../kafka/partition/model/PluginScale.java | 49 ----- .../kafka/partition/model/PromotePlugin.java | 73 ------- .../PartitionPlanReplacementTest.java | 167 --------------- .../model/PartitionPlanJsonTest.java | 16 ++ 7 files changed, 35 insertions(+), 559 deletions(-) delete mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java delete mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java delete mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java delete mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java delete mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java index 200d0bf9de3..de51403863d 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -29,6 +29,7 @@ import org.apache.ranger.audit.server.AuditServerConstants; import org.springframework.stereotype.Component; +import java.util.Collections; import java.util.Properties; import java.util.Set; @@ -41,21 +42,25 @@ public class PartitionPlanService { private final PartitionPlanHolder holder; private final PartitionPlanRegistryFactory registryFactory; private final KafkaAuditTopicPartitionGrower auditTopicPartitionGrower; + private final boolean dynamicPartitionPlanEnabled; + private final Set partitionPlanAdminUsers; public PartitionPlanService() { this(AuditServerConfig.getInstance().getProperties(), PartitionPlanHolder.getInstance(), new PartitionPlanRegistryFactory(), new KafkaAuditTopicPartitionGrower()); } PartitionPlanService(Properties configProps, PartitionPlanHolder holder, PartitionPlanRegistryFactory registryFactory, KafkaAuditTopicPartitionGrower auditTopicPartitionGrower) { - this.configProps = configProps; - this.holder = holder; - this.registryFactory = registryFactory; - this.auditTopicPartitionGrower = auditTopicPartitionGrower; + this.configProps = configProps; + this.holder = holder; + this.registryFactory = registryFactory; + this.auditTopicPartitionGrower = auditTopicPartitionGrower; + this.dynamicPartitionPlanEnabled = PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configProps, INGESTOR_PROP_PREFIX); + this.partitionPlanAdminUsers = cachePartitionPlanAdminUsers(configProps); } /** Returns whether dynamic partition-plan mode is enabled in ingestor configuration. */ public boolean isDynamicPartitionPlanEnabled() { - return PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(configProps, INGESTOR_PROP_PREFIX); + return dynamicPartitionPlanEnabled; } /** Returns the plan currently installed in memory on this ingestor pod. */ @@ -109,7 +114,15 @@ public PartitionPlan updatePlugin(String pluginId, UpdatePlugin updatePluginRequ /** Returns configured admin short usernames for partition-plan REST (empty = not restricted beyond authentication). */ public Set getPartitionPlanAdminUsers() { - return PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); + return partitionPlanAdminUsers; + } + + private static Set cachePartitionPlanAdminUsers(Properties configProps) { + Set adminUsers = PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); + if (adminUsers.isEmpty()) { + return adminUsers; + } + return Collections.unmodifiableSet(adminUsers); } /** Validates version, grows the audit topic if needed, writes the plan, and reloads memory. */ diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java deleted file mode 100644 index b39331eaa47..00000000000 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/OnboardService.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.audit.producer.kafka.partition.model; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.io.Serializable; -import java.util.Collections; -import java.util.List; - -@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class OnboardService implements Serializable { - private final String serviceName; - private final String pluginId; - private final int partitionCount; - private final List allowedUsers; - private final int expectedVersion; - - @JsonCreator - public OnboardService(@JsonProperty("serviceName") String serviceName, @JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("allowedUsers") List allowedUsers, @JsonProperty("expectedVersion") int expectedVersion) { - this.serviceName = serviceName; - this.pluginId = pluginId; - this.partitionCount = partitionCount; - this.allowedUsers = allowedUsers == null ? Collections.emptyList() : List.copyOf(allowedUsers); - this.expectedVersion = expectedVersion; - } - - public String getServiceName() { - return serviceName; - } - - public String getPluginId() { - return pluginId; - } - - public int getPartitionCount() { - return partitionCount; - } - - public List getAllowedUsers() { - return allowedUsers; - } - - public int getExpectedVersion() { - return expectedVersion; - } -} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java deleted file mode 100644 index 479377c1b95..00000000000 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanReplacement.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.audit.producer.kafka.partition.model; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.lang3.StringUtils; -import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; - -import java.io.Serializable; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Partial partition-plan update (REST PATCH). Omitted or empty fields are inherited from the current plan. - * {@code plugins} and {@code services} merge only entries whose keys are not already present. - */ -@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class PartitionPlanReplacement implements Serializable { - private final int expectedVersion; - private final Integer topicPartitionCount; - private final Map plugins; - private final PluginPartitionAssignment buffer; - private final Map services; - - @JsonCreator - public PartitionPlanReplacement(@JsonProperty("expectedVersion") int expectedVersion, - @JsonProperty("topicPartitionCount") Integer topicPartitionCount, - @JsonProperty("plugins") Map plugins, - @JsonProperty("buffer") PluginPartitionAssignment buffer, - @JsonProperty("services") Map services) { - this.expectedVersion = expectedVersion; - this.topicPartitionCount = topicPartitionCount; - this.plugins = copyPlugins(plugins); - this.buffer = buffer; - this.services = copyServices(services); - } - - public PartitionPlanReplacement(int expectedVersion, int topicPartitionCount, - Map plugins, PluginPartitionAssignment buffer, - Map services) { - this(expectedVersion, Integer.valueOf(topicPartitionCount), plugins, buffer, services); - } - - private static Map copyPlugins(Map plugins) { - if (plugins == null || plugins.isEmpty()) { - return null; - } - return Collections.unmodifiableMap(new LinkedHashMap<>(plugins)); - } - - private static Map copyServices(Map services) { - if (services == null || services.isEmpty()) { - return null; - } - return Collections.unmodifiableMap(new LinkedHashMap<>(services)); - } - - public int getExpectedVersion() { - return expectedVersion; - } - - public Integer getTopicPartitionCount() { - return topicPartitionCount; - } - - public Map getPlugins() { - return plugins; - } - - public PluginPartitionAssignment getBuffer() { - return buffer; - } - - public Map getServices() { - return services; - } - - /** True when the request carries at least one field to merge into the current plan. */ - public boolean hasMergeDelta() { - return hasTopicPartitionCountDelta() || hasPluginsDelta() || hasBufferDelta() || hasServicesDelta(); - } - - public boolean hasTopicPartitionCountDelta() { - return topicPartitionCount != null && topicPartitionCount >= 1; - } - - public boolean hasPluginsDelta() { - return plugins != null && !plugins.isEmpty(); - } - - public boolean hasBufferDelta() { - return buffer != null && !buffer.getPartitions().isEmpty(); - } - - public boolean hasServicesDelta() { - return services != null && !services.isEmpty(); - } - - /** Merges this delta into {@code currentPlan} and returns the proposed next plan (version not incremented). */ - public PartitionPlan toMergedPlan(PartitionPlan currentPlan, String updatedBy) { - if (currentPlan == null) { - throw new PartitionPlanException("Current partition plan is required"); - } - - int mergedTopicPartitionCount = hasTopicPartitionCountDelta() - ? topicPartitionCount - : currentPlan.getTopicPartitionCount(); - - Map mergedPlugins = new LinkedHashMap<>(currentPlan.getPlugins()); - List newlyAddedPluginAssignments = new ArrayList<>(); - if (hasPluginsDelta()) { - for (Map.Entry pluginDeltaEntry : plugins.entrySet()) { - String pluginId = pluginDeltaEntry.getKey(); - if (StringUtils.isBlank(pluginId)) { - throw new PartitionPlanException("Plugin id is required"); - } - if (mergedPlugins.containsKey(pluginId)) { - PluginPartitionAssignment existingAssignment = mergedPlugins.get(pluginId); - PluginPartitionAssignment requestedAssignment = pluginDeltaEntry.getValue(); - if (existingAssignment.equals(requestedAssignment)) { - continue; - } - throw new PartitionPlanException( - "Plugin '" + pluginId + "' already exists with different partition assignment; use PATCH /partition-plan/plugins/{pluginId} to grow it"); - } - mergedPlugins.put(pluginId, pluginDeltaEntry.getValue()); - newlyAddedPluginAssignments.add(pluginDeltaEntry.getValue()); - } - } - - PluginPartitionAssignment mergedBuffer = hasBufferDelta() ? buffer : currentPlan.getBuffer(); - - Map mergedServices = new LinkedHashMap<>(currentPlan.getServices()); - if (hasServicesDelta()) { - for (Map.Entry serviceDeltaEntry : services.entrySet()) { - String serviceRepoName = serviceDeltaEntry.getKey(); - if (StringUtils.isBlank(serviceRepoName)) { - throw new PartitionPlanException("Service repo name is required"); - } - mergedServices.put(serviceRepoName.trim(), serviceDeltaEntry.getValue()); - } - } - - if (!newlyAddedPluginAssignments.isEmpty()) { - mergedBuffer = removeAssignedPartitionsFromBuffer(mergedBuffer, newlyAddedPluginAssignments); - } - - return PartitionPlan.builder() - .topic(currentPlan.getTopic()) - .topicPartitionCount(mergedTopicPartitionCount) - .plugins(mergedPlugins) - .buffer(mergedBuffer) - .services(mergedServices) - .updatedAt(Instant.now().toString()) - .updatedBy(updatedBy) - .build(); - } - - private static PluginPartitionAssignment removeAssignedPartitionsFromBuffer( - PluginPartitionAssignment bufferAssignment, Iterable newPluginAssignments) { - List remainingBufferPartitionIds = new ArrayList<>(bufferAssignment.getPartitions()); - for (PluginPartitionAssignment pluginAssignment : newPluginAssignments) { - for (Integer partitionId : pluginAssignment.getPartitions()) { - remainingBufferPartitionIds.remove(partitionId); - } - } - return new PluginPartitionAssignment(remainingBufferPartitionIds); - } -} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java deleted file mode 100644 index 73e493f7b39..00000000000 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PluginScale.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.audit.producer.kafka.partition.model; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.io.Serializable; - -@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class PluginScale implements Serializable { - private final int additionalPartitions; - private final int expectedVersion; - - @JsonCreator - public PluginScale(@JsonProperty("additionalPartitions") int additionalPartitions, @JsonProperty("expectedVersion") int expectedVersion) { - this.additionalPartitions = additionalPartitions; - this.expectedVersion = expectedVersion; - } - - public int getAdditionalPartitions() { - return additionalPartitions; - } - - public int getExpectedVersion() { - return expectedVersion; - } -} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java deleted file mode 100644 index ef8d347d373..00000000000 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/PromotePlugin.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.audit.producer.kafka.partition.model; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.io.Serializable; -import java.util.Collections; -import java.util.List; - -@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class PromotePlugin implements Serializable { - private final String pluginId; - private final int partitionCount; - private final int expectedVersion; - private final String repo; - private final List allowedUsers; - - @JsonCreator - public PromotePlugin(@JsonProperty("pluginId") String pluginId, @JsonProperty("partitionCount") int partitionCount, @JsonProperty("expectedVersion") int expectedVersion, @JsonProperty("repo") String repo, @JsonProperty("allowedUsers") List allowedUsers) { - this.pluginId = pluginId; - this.partitionCount = partitionCount; - this.expectedVersion = expectedVersion; - this.repo = repo; - this.allowedUsers = allowedUsers == null ? Collections.emptyList() : List.copyOf(allowedUsers); - } - - public PromotePlugin(String pluginId, int partitionCount, int expectedVersion) { - this(pluginId, partitionCount, expectedVersion, null, null); - } - - public String getPluginId() { - return pluginId; - } - - public int getPartitionCount() { - return partitionCount; - } - - public int getExpectedVersion() { - return expectedVersion; - } - - public String getRepo() { - return repo; - } - - public List getAllowedUsers() { - return allowedUsers; - } -} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java deleted file mode 100644 index 481e73ec500..00000000000 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanReplacementTest.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.audit.producer.kafka.partition; - -import org.apache.ranger.audit.producer.kafka.partition.exception.PartitionPlanException; -import org.apache.ranger.audit.producer.kafka.partition.model.OnboardPlugin; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; -import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlanReplacement; -import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; -import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; -import org.apache.ranger.audit.producer.kafka.partition.model.UpdatePlugin; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class PartitionPlanReplacementTest { - private static final PartitionPlan CURRENT = PartitionPlan.builder() - .topic("ranger_audits") - .version(1) - .topicPartitionCount(9) - .plugins(Map.of( - "hdfs", PluginPartitionAssignment.of(0, 1, 2), - "hiveServer2", PluginPartitionAssignment.of(3, 4, 5))) - .buffer(PluginPartitionAssignment.of(6, 7, 8)) - .services(Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs"))) - .build(); - - @Test - public void testMergeInheritsEmptyOptionalFields() { - PartitionPlanReplacement update = new PartitionPlanReplacement(1, null, null, null, null); - - PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); - - assertEquals(9, merged.getTopicPartitionCount()); - assertEquals(CURRENT.getPlugins(), merged.getPlugins()); - assertEquals(CURRENT.getBuffer(), merged.getBuffer()); - assertEquals(CURRENT.getServices(), merged.getServices()); - } - - @Test - public void testMergeAddsOnlyNewPluginAndServiceEntries() { - PartitionPlanReplacement update = new PartitionPlanReplacement( - 1, - null, - Map.of("trino", PluginPartitionAssignment.of(6, 7, 8)), - null, - Map.of("dev_trino", ServiceAllowlistEntry.ofUsers("trino"))); - - PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); - - assertTrue(merged.getPlugins().containsKey("trino")); - assertTrue(merged.getServices().containsKey("dev_trino")); - assertTrue(merged.getPlugins().containsKey("hdfs")); - assertTrue(merged.getServices().containsKey("dev_hdfs")); - } - - @Test - public void testHasMergeDeltaRequiresNonEmptyField() { - assertFalse(new PartitionPlanReplacement(1, null, null, null, null).hasMergeDelta()); - assertTrue(new PartitionPlanReplacement(1, 12, null, null, null).hasMergeDelta()); - assertTrue(new PartitionPlanReplacement(1, null, Map.of("trino", PluginPartitionAssignment.of(6)), null, null).hasMergeDelta()); - } - - @Test - public void testMergeUpsertsExistingServiceAllowlist() { - PartitionPlanReplacement update = new PartitionPlanReplacement( - 1, - null, - null, - null, - Map.of("dev_hdfs", ServiceAllowlistEntry.ofUsers("hdfs", "backup"))); - - PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); - - assertIterableEquals(List.of("hdfs", "backup"), merged.getServices().get("dev_hdfs").getAllowedUsers()); - } - - @Test - public void testMergeExistingPluginWithSameAssignmentIsNoOpDelta() { - PartitionPlanReplacement update = new PartitionPlanReplacement( - 1, - null, - Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2)), - null, - null); - - PartitionPlan merged = update.toMergedPlan(CURRENT, "ops"); - - assertTrue(CURRENT.sameContentAs(merged)); - } - - @Test - public void testMergeExistingPluginWithDifferentAssignmentFails() { - PartitionPlanReplacement update = new PartitionPlanReplacement( - 1, - null, - Map.of("hdfs", PluginPartitionAssignment.of(0, 1)), - null, - null); - - PartitionPlanException error = assertThrows(PartitionPlanException.class, () -> update.toMergedPlan(CURRENT, "ops")); - - assertTrue(error.getMessage().contains("different partition assignment")); - } - - @Test - public void testSameContentAsIgnoresVersionAndMetadata() { - PartitionPlan withMetadata = CURRENT.toBuilder().version(99).updatedAt("later").updatedBy("other").build(); - - assertTrue(CURRENT.sameContentAs(withMetadata)); - } - - @Test - public void testRequestValidatorRejectsBlankOnboardPluginId() { - assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("", 2, 1))); - } - - @Test - public void testRequestValidatorRejectsOnboardMissingServices() { - PartitionPlanException error = assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1))); - - assertTrue(error.getMessage().contains("services are required")); - } - - @Test - public void testRequestValidatorRejectsOnboardServiceWithoutAllowedUsers() { - Map services = Map.of("dev_trino", ServiceAllowlistEntry.ofUsers(List.of(" "))); - assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateOnboardPlugin(new OnboardPlugin("trino", 2, 1, services))); - } - - @Test - public void testRequestValidatorRejectsUpdateWithZeroAdditionalPartitions() { - assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, 0, null, null, null))); - } - - @Test - public void testRequestValidatorRejectsUpdateWithoutMutationDelta() { - assertThrows(PartitionPlanException.class, - () -> PartitionPlanRequestValidator.validateUpdatePlugin("hiveServer2", new UpdatePlugin(1, null, null, null, null))); - } -} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java index 2778bd833e3..b4fb2aa9bbf 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/model/PartitionPlanJsonTest.java @@ -28,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class PartitionPlanJsonTest { @Test @@ -57,6 +58,21 @@ public void testRoundTripPreservesServices() { assertIterableEquals(List.of("hive"), parsed.getServices().get("dev_hive").getAllowedUsers()); } + @Test + public void testSameContentAsIgnoresVersionAndMetadata() { + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(1) + .topicPartitionCount(6) + .plugins(Map.of("hdfs", PluginPartitionAssignment.of(0, 1, 2))) + .buffer(PluginPartitionAssignment.of(3, 4, 5)) + .services(Map.of("dev_hive", ServiceAllowlistEntry.ofUsers("hive"))) + .build(); + PartitionPlan withMetadata = plan.toBuilder().version(99).updatedAt("later").updatedBy("other").build(); + + assertTrue(plan.sameContentAs(withMetadata)); + } + @Test public void testFromJsonRejectsInvalidPlan() { assertThrows(PartitionPlanException.class, () -> PartitionPlan.fromJson("{\"topic\":\"ranger_audits\",\"version\":1}")); From ba6135493297373a2d14122c958037188d3f0931 Mon Sep 17 00:00:00 2001 From: ramk Date: Wed, 24 Jun 2026 14:17:23 +0530 Subject: [PATCH 09/10] RANGER-5655: Use single return per method in partition-plan code Refactor partition-plan helpers and AuditREST partition-plan paths to match Ranger review style with one return statement per method. --- .../partition/AuthToLocalRuleComposer.java | 7 +- .../partition/PartitionPlanAllocator.java | 122 +++++++++++------- .../kafka/partition/PartitionPlanHolder.java | 26 ++-- .../partition/PartitionPlanKafkaConfig.java | 42 +++--- .../PartitionPlanRequestValidator.java | 4 + .../kafka/partition/PartitionPlanService.java | 9 +- .../partition/ServiceAllowlistBootstrap.java | 17 ++- .../partition/ServiceAllowlistResolver.java | 26 ++-- .../kafka/partition/model/UpdatePlugin.java | 19 +-- .../apache/ranger/audit/rest/AuditREST.java | 74 ++++++----- 10 files changed, 212 insertions(+), 134 deletions(-) diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java index 95184a6f296..081bb6b790d 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java @@ -150,11 +150,14 @@ public void setPartitionPlanTopicExistsTestOverride(Boolean topicExists) { } private boolean isPartitionPlanTopicPresent(Properties ingestorProperties, String ingestorPropertyPrefix) { + boolean ret = AuditMessageQueueUtils.partitionPlanTopicExists(ingestorProperties, ingestorPropertyPrefix); Boolean topicExistsOverride = partitionPlanTopicExistsTestOverride; + if (topicExistsOverride != null) { - return topicExistsOverride; + ret = topicExistsOverride; } - return AuditMessageQueueUtils.partitionPlanTopicExists(ingestorProperties, ingestorPropertyPrefix); + + return ret; } /** diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java index 54ca5480d42..07104017a66 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanAllocator.java @@ -149,54 +149,82 @@ public static PartitionPlan scalePlugin(PartitionPlan current, String pluginId, * True when plugin onboard with {@code partitionCount} and optional services already matches the current plan. */ public static boolean isOnboardAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, Map servicesMap) { - if (current == null || !pluginHasPartitionCount(current, pluginId, partitionCount)) { - return false; - } - if (servicesMap == null || servicesMap.isEmpty()) { - return true; - } - for (Map.Entry entry : servicesMap.entrySet()) { - String repo = entry.getKey().trim(); - ServiceAllowlistEntry existing = current.getServices().get(repo); - ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); - if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { - return false; + boolean ret = false; + + if (current != null && pluginHasPartitionCount(current, pluginId, partitionCount)) { + ret = true; + + if (servicesMap != null && !servicesMap.isEmpty()) { + for (Map.Entry entry : servicesMap.entrySet()) { + String repo = entry.getKey().trim(); + ServiceAllowlistEntry existing = current.getServices().get(repo); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } } } - return true; + + return ret; } /** * True when a service-only update request is already satisfied (scale mutations are never treated as no-op). */ public static boolean isUpdateAlreadyApplied(PartitionPlan current, String pluginId, UpdatePlugin updateRequest) { - if (current == null || updateRequest == null) { - return false; - } - Integer additionalPartitions = updateRequest.getAdditionalPartitions(); - if (additionalPartitions != null && additionalPartitions >= 1) { - return false; - } - for (String repo : updateRequest.getRemoveServices()) { - if (current.getServices().containsKey(repo.trim())) { - return false; - } - } - for (Map.Entry entry : updateRequest.getAddServices().entrySet()) { - ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); - ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); - if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { - return false; - } - } - for (Map.Entry entry : updateRequest.getUpdateServices().entrySet()) { - ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); - ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); - if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { - return false; + boolean ret = false; + + if (current != null && updateRequest != null) { + Integer additionalPartitions = updateRequest.getAdditionalPartitions(); + + if (additionalPartitions == null || additionalPartitions < 1) { + ret = true; + + for (String repo : updateRequest.getRemoveServices()) { + if (current.getServices().containsKey(repo.trim())) { + ret = false; + + break; + } + } + + if (ret) { + for (Map.Entry entry : updateRequest.getAddServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } + } + + if (ret) { + for (Map.Entry entry : updateRequest.getUpdateServices().entrySet()) { + ServiceAllowlistEntry existing = current.getServices().get(entry.getKey().trim()); + ServiceAllowlistEntry expected = withPluginId(entry.getValue(), pluginId); + + if (existing == null || !serviceEntryMatches(existing, expected, pluginId)) { + ret = false; + + break; + } + } + } + + if (ret) { + ret = updateRequest.hasMutationDelta(); + } } } - return updateRequest.hasMutationDelta(); + + return ret; } /** @@ -204,14 +232,18 @@ public static boolean isUpdateAlreadyApplied(PartitionPlan current, String plugi * the service allowlist already matches {@code allowedUsers}. */ public static boolean isPromoteAlreadyApplied(PartitionPlan current, String pluginId, int partitionCount, String repo, List allowedUsers) { - if (current == null || !pluginHasPartitionCount(current, pluginId, partitionCount)) { - return false; - } - if (StringUtils.isNotBlank(repo)) { - ServiceAllowlistEntry existing = current.getServices().get(repo.trim()); - return existing != null && existing.hasSameAllowedUsers(allowedUsers); + boolean ret = false; + + if (current != null && pluginHasPartitionCount(current, pluginId, partitionCount)) { + if (StringUtils.isNotBlank(repo)) { + ServiceAllowlistEntry existing = current.getServices().get(repo.trim()); + ret = existing != null && existing.hasSameAllowedUsers(allowedUsers); + } else { + ret = true; + } } - return true; + + return ret; } /** Applies a merged plan with append-only checks against the current plan. */ diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java index c89a391b40c..cf1d890eb43 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java @@ -67,18 +67,22 @@ public void install(PartitionPlan plan, Integer kafkaPartitionCount) { * allowlist union ({@link AuthToLocalRuleComposer#collectAllowedUserShortNames}). */ public Set getAllowedUsersForService(String serviceName) { - PartitionPlan plan = planRef.get(); - if (plan == null || plan.getServices() == null || plan.getServices().isEmpty()) { - return null; - } - ServiceAllowlistEntry entry = plan.getServices().get(serviceName); - if (entry == null) { - return null; - } - if (entry.getAllowedUsers().isEmpty()) { - return Collections.emptySet(); + Set ret = null; + PartitionPlan plan = planRef.get(); + + if (plan != null && plan.getServices() != null && !plan.getServices().isEmpty()) { + ServiceAllowlistEntry entry = plan.getServices().get(serviceName); + + if (entry != null) { + if (entry.getAllowedUsers().isEmpty()) { + ret = Collections.emptySet(); + } else { + ret = Collections.unmodifiableSet(new LinkedHashSet<>(entry.getAllowedUsers())); + } + } } - return Collections.unmodifiableSet(new LinkedHashSet<>(entry.getAllowedUsers())); + + return ret; } /** Clears holder state between unit tests. */ diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java index 486ed2ce516..8c6275206b5 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanKafkaConfig.java @@ -49,30 +49,38 @@ public static boolean isDynamicPartitionPlanEnabled(Properties props, String pro /** Resolves short usernames allowed to call partition-plan admin REST (empty = any authenticated principal). */ public static Set resolvePartitionPlanAdminUsers(Properties props, String propPrefix) { - String configured = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_ALLOWED_USERS, ""); - if (configured == null || configured.isBlank()) { - return Collections.emptySet(); - } - Set users = new LinkedHashSet<>(); - for (String user : configured.split(",")) { - if (user != null && !user.isBlank()) { - users.add(user.trim()); + Set ret = Collections.emptySet(); + String configured = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_ALLOWED_USERS, ""); + + if (configured != null && !configured.isBlank()) { + Set users = new LinkedHashSet<>(); + + for (String user : configured.split(",")) { + if (user != null && !user.isBlank()) { + users.add(user.trim()); + } } + ret = users; } - return users; + + return ret; } /** Returns whether the Kafka producer partitioner should use the in-memory dynamic plan. */ public static boolean isDynamicPartitionPlanEnabled(Map configs, String ingestorPropPrefix) { - String key = ingestorPropPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED; - Object val = configs.get(key); - if (val == null) { - return false; - } - if (val instanceof Boolean) { - return (Boolean) val; + boolean ret = false; + String key = ingestorPropPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED; + Object val = configs.get(key); + + if (val != null) { + if (val instanceof Boolean) { + ret = (Boolean) val; + } else { + ret = Boolean.parseBoolean(val.toString()); + } } - return Boolean.parseBoolean(val.toString()); + + return ret; } /** Resolves how often each ingestor pod reloads the plan from Kafka. */ diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java index d32665b3195..a1ea6475358 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanRequestValidator.java @@ -123,13 +123,17 @@ private static void validateNonEmptyAllowedUsers(List allowedUsers) { if (allowedUsers == null || allowedUsers.isEmpty()) { throw new PartitionPlanException("allowedUsers are required"); } + boolean hasNonBlankUser = false; + for (String allowedUserShortName : allowedUsers) { if (StringUtils.isNotBlank(allowedUserShortName)) { hasNonBlankUser = true; + break; } } + if (!hasNonBlankUser) { throw new PartitionPlanException("allowedUsers must contain at least one non-blank username"); } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java index de51403863d..ce4d3c8651b 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -119,10 +119,13 @@ public Set getPartitionPlanAdminUsers() { private static Set cachePartitionPlanAdminUsers(Properties configProps) { Set adminUsers = PartitionPlanKafkaConfig.resolvePartitionPlanAdminUsers(configProps, INGESTOR_PROP_PREFIX); - if (adminUsers.isEmpty()) { - return adminUsers; + Set ret = adminUsers; + + if (!adminUsers.isEmpty()) { + ret = Collections.unmodifiableSet(adminUsers); } - return Collections.unmodifiableSet(adminUsers); + + return ret; } /** Validates version, grows the audit topic if needed, writes the plan, and reloads memory. */ diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java index c7d8aca0df2..1ef0ef77315 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistBootstrap.java @@ -80,14 +80,17 @@ public static Map loadAllowlistsFromServerConfig( */ public static PartitionPlan mergeSiteXmlAllowlistsWhenPlanServicesMissing( PartitionPlan partitionPlan, Properties ingestorProperties) { - if (partitionPlan == null || (partitionPlan.getServices() != null && !partitionPlan.getServices().isEmpty())) { - return partitionPlan; - } - Map siteXmlAllowlistEntries = loadAllowlistsFromProperties(ingestorProperties); - if (siteXmlAllowlistEntries.isEmpty()) { - return partitionPlan; + PartitionPlan ret = partitionPlan; + + if (partitionPlan != null && (partitionPlan.getServices() == null || partitionPlan.getServices().isEmpty())) { + Map siteXmlAllowlistEntries = loadAllowlistsFromProperties(ingestorProperties); + + if (!siteXmlAllowlistEntries.isEmpty()) { + ret = partitionPlan.toBuilder().services(siteXmlAllowlistEntries).build(); + } } - return partitionPlan.toBuilder().services(siteXmlAllowlistEntries).build(); + + return ret; } private static List parseAllowedUserShortNames(String allowedUsersPropertyValue) { diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java index 8b0b09bceb5..59080afc212 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/ServiceAllowlistResolver.java @@ -47,16 +47,24 @@ private ServiceAllowlistResolver() { * @param holder partition-plan registry; per-repo {@code services[serviceName].allowedUsers} */ public static boolean isAllowedServiceUser(String serviceName, String userName, boolean dynamicPartitionPlanEnabled, PartitionPlanHolder holder, Map> staticAllowedUsersByService) { - if (StringUtils.isBlank(serviceName) || StringUtils.isBlank(userName)) { - return false; - } - if (dynamicPartitionPlanEnabled && holder != null) { - Set registryUsers = holder.getAllowedUsersForService(serviceName); - if (registryUsers != null) { - return registryUsers.contains(userName); + boolean ret = false; + + if (!StringUtils.isBlank(serviceName) && !StringUtils.isBlank(userName)) { + if (dynamicPartitionPlanEnabled && holder != null) { + Set registryUsers = holder.getAllowedUsersForService(serviceName); + + if (registryUsers != null) { + ret = registryUsers.contains(userName); + } else { + Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; + ret = allowedUsers != null && allowedUsers.contains(userName); + } + } else { + Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; + ret = allowedUsers != null && allowedUsers.contains(userName); } } - Set allowedUsers = staticAllowedUsersByService != null ? staticAllowedUsersByService.get(serviceName) : null; - return allowedUsers != null && allowedUsers.contains(userName); + + return ret; } } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java index 535971c4e76..1eb3cf4ec9c 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/model/UpdatePlugin.java @@ -78,15 +78,18 @@ public List getRemoveServices() { /** True when at least one mutation field is present. */ public boolean hasMutationDelta() { + boolean ret = false; + if (additionalPartitions != null && additionalPartitions >= 1) { - return true; - } - if (!addServices.isEmpty()) { - return true; + ret = true; + } else if (!addServices.isEmpty()) { + ret = true; + } else if (!updateServices.isEmpty()) { + ret = true; + } else if (!removeServices.isEmpty()) { + ret = true; } - if (!updateServices.isEmpty()) { - return true; - } - return !removeServices.isEmpty(); + + return ret; } } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index c39fc1360cd..a55b5a34cf7 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -377,32 +377,42 @@ private Response partitionPlanDisabled(String operation) { * When unset, any authenticated principal may call partition-plan (backward compatible). */ private Response authorizePartitionPlanAdmin(HttpServletRequest request, String operation) { - String user = getAuthenticatedUser(request); + Response ret = null; + String user = getAuthenticatedUser(request); + if (StringUtils.isBlank(user)) { LOG.error("{} rejected: authentication required", operation); - return Response.status(Response.Status.UNAUTHORIZED).entity(buildErrorResponse("Authentication required")).build(); - } - Set adminUsers = partitionPlanService.getPartitionPlanAdminUsers(); - if (!adminUsers.isEmpty() && !adminUsers.contains(user)) { - LOG.error("{} rejected: user '{}' is not in partition plan admin allowlist", operation, user); - return Response.status(Response.Status.FORBIDDEN).entity(buildErrorResponse("User is not authorized to manage partition plan")).build(); + ret = Response.status(Response.Status.UNAUTHORIZED).entity(buildErrorResponse("Authentication required")).build(); + } else { + Set adminUsers = partitionPlanService.getPartitionPlanAdminUsers(); + + if (!adminUsers.isEmpty() && !adminUsers.contains(user)) { + LOG.error("{} rejected: user '{}' is not in partition plan admin allowlist", operation, user); + ret = Response.status(Response.Status.FORBIDDEN).entity(buildErrorResponse("User is not authorized to manage partition plan")).build(); + } } - return null; + + return ret; } /** Maps service/infrastructure failures to 503; client validation mistakes to 400. */ private static Response.Status resolvePartitionPlanErrorStatus(PartitionPlanException error) { + Response.Status ret = Response.Status.BAD_REQUEST; + if (error.getCause() != null) { - return Response.Status.SERVICE_UNAVAILABLE; - } - String message = error.getMessage(); - if (message != null && (message.contains("Partition plan is not loaded in memory") - || message.contains("Partition plan disappeared during update") - || message.contains("No partition plan found in Kafka") - || message.contains("Mandatory read-back failed"))) { - return Response.Status.SERVICE_UNAVAILABLE; + ret = Response.Status.SERVICE_UNAVAILABLE; + } else { + String message = error.getMessage(); + + if (message != null && (message.contains("Partition plan is not loaded in memory") + || message.contains("Partition plan disappeared during update") + || message.contains("No partition plan found in Kafka") + || message.contains("Mandatory read-back failed"))) { + ret = Response.Status.SERVICE_UNAVAILABLE; + } } - return Response.Status.BAD_REQUEST; + + return ret; } /** Records the authenticated admin user on plan mutations. */ @@ -452,24 +462,24 @@ private String getAuthenticatedUser(HttpServletRequest request) { * For JWT or basic auth, the username is already in short form and returned as-is. */ private String applyAuthToLocal(String principal) { - if (StringUtils.isEmpty(principal)) { - return principal; - } + String ret = principal; - // Check if this looks like a Kerberos principal (has @ or /) - if (!principal.contains("@") && !principal.contains("/")) { - LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); - return principal; - } - - try { - KerberosName kerberosName = new KerberosName(principal); + if (StringUtils.isNotEmpty(principal)) { + // Check if this looks like a Kerberos principal (has @ or /) + if (!principal.contains("@") && !principal.contains("/")) { + LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); + } else { + try { + KerberosName kerberosName = new KerberosName(principal); - return kerberosName.getShortName(); - } catch (Exception e) { - LOG.warn("Failed to apply auth_to_local rules to principal '{}': {}. Using original principal.", principal, e.getMessage()); - return principal; + ret = kerberosName.getShortName(); + } catch (Exception e) { + LOG.warn("Failed to apply auth_to_local rules to principal '{}': {}. Using original principal.", principal, e.getMessage()); + } + } } + + return ret; } /** From df5a6e28f9784dc585a55b7d4e844d84b3615b82 Mon Sep 17 00:00:00 2001 From: ramk Date: Sat, 27 Jun 2026 18:00:48 +0530 Subject: [PATCH 10/10] RANGER-5655: Address PR #1032 review feedback on partition-plan code Use Math.floorMod for buffer hash routing, align PartitionPlanHolder Javadoc with validator rules, and update configured.plugins REST doc. Co-authored-by: Cursor --- .../apache/ranger/audit/producer/kafka/AuditPartitioner.java | 2 +- .../audit/producer/kafka/partition/PartitionPlanHolder.java | 3 ++- .../src/main/resources/conf/ranger-audit-ingestor-site.xml | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java index f9a0c837368..081e5a544ec 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/AuditPartitioner.java @@ -232,7 +232,7 @@ private int partitionFromPlan(String appId, int kafkaClusterPartitionCount) { * Used for buffer-pool routing and for plan-not-loaded fallback. */ private static int hashAppIdToPartitionIndex(String appId, int slotCount) { - return Math.abs(appId.hashCode() % slotCount); + return Math.floorMod(appId.hashCode(), slotCount); } /** diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java index cf1d890eb43..325d8386933 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanHolder.java @@ -61,7 +61,8 @@ public void install(PartitionPlan plan, Integer kafkaPartitionCount) { * Returns allowed short usernames for a service repo from the in-memory registry document. * {@code null} when the plan has no {@code services} block, or when the repo is not present * in the plan (caller should fall back to static XML). - * Returns an empty set when the repo is present with an explicit empty allowlist (deny all). + * Returns a non-empty set when the repo is present; {@link PartitionPlanValidator} rejects + * plans whose {@code allowedUsers} list is empty at install time. * *

Used by {@link ServiceAllowlistResolver} for per-repo POST authorization — not for the global * allowlist union ({@link AuthToLocalRuleComposer#collectAllowedUserShortNames}). diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 7f68a020fe7..52bb6594dff 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -360,8 +360,7 @@ Empty (recommended greenfield): - Static mode: Kafka default hash partitioner using kafka.topic.partitions below. - Dynamic mode (kafka.partition.plan.dynamic.enabled=true): first bootstrap plan uses all - kafka.topic.partitions as buffer; onboard plugins via POST /api/audit/partition-plan/services - or promote without editing this list. + kafka.topic.partitions as buffer; onboard plugins via POST /api/audit/partition-plan/plugins. Non-empty (static / bootstrap seed): - Uses AuditPartitioner: total partitions = sum(per-plugin counts) + kafka.topic.partitions.buffer.