From 4cd4c246f4c584b70f456446d0b9c24f42cf2552 Mon Sep 17 00:00:00 2001 From: Anant Sahrawat Date: Fri, 23 Oct 2026 16:55:52 +0530 Subject: [PATCH 1/2] Resolve EQUALITY keys via hash lookup during search Search used to evaluate every key of the inverted index against the request to find matching keys, so its cost grew with the number of distinct keys even when few criteria matched. EQUALITY keys with string, boolean or standard numeric values are now bucketed by (path, preOperations) and normalised value, and resolved with a hash lookup; all other keys are still evaluated one by one. Value normalisation mirrors Utils.compare, so results are unchanged. The lookup is derived lazily from each posting-list map and rebuilt when the indexers replace the map. It is not serialised. The behaviour can be turned off with MustangEngine.builder().equalityKeyLookupEnabled(false). Adds parity tests against the previous scan behaviour and an equality-heavy JMH benchmark, and parameterises the existing benchmarks over the flag. --- .../MustangEqualitySearchBenchmark.java | 161 ++++++++ .../MustangOnlyCNFSearchBenchmark.java | 4 + .../MustangOnlyDNFSearchBenchmark.java | 4 + .../benchmarks/MustangSearchBenchmark.java | 4 + .../com/phonepe/mustang/MustangEngine.java | 10 +- .../phonepe/mustang/search/SearchFacade.java | 9 + .../search/handler/CriteriaSearchHandler.java | 4 + .../mustang/search/matcher/CNFMatcher.java | 16 +- .../mustang/search/matcher/DNFMatcher.java | 16 +- .../search/matcher/MatchingKeyResolver.java | 88 ++++ .../search/EqualityKeyLookupSearchTest.java | 380 ++++++++++++++++++ .../mustang/index/core/EqualityKeyLookup.java | 161 ++++++++ .../mustang/index/core/InvertedIndex.java | 34 ++ 13 files changed, 871 insertions(+), 20 deletions(-) create mode 100644 benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangEqualitySearchBenchmark.java create mode 100644 mustang-core/src/main/java/com/phonepe/mustang/search/matcher/MatchingKeyResolver.java create mode 100644 mustang-core/src/test/java/com/phonepe/mustang/search/EqualityKeyLookupSearchTest.java create mode 100644 mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java diff --git a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangEqualitySearchBenchmark.java b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangEqualitySearchBenchmark.java new file mode 100644 index 0000000..45c003c --- /dev/null +++ b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangEqualitySearchBenchmark.java @@ -0,0 +1,161 @@ +/** + * Copyright (c) 2022 PhonePe India Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.phonepe.mustang.benchmark.benchmarks; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import com.phonepe.mustang.MustangEngine; +import com.phonepe.mustang.benchmark.utils.Utils; +import com.phonepe.mustang.common.RequestContext; +import com.phonepe.mustang.composition.impl.Conjunction; +import com.phonepe.mustang.criteria.Criteria; +import com.phonepe.mustang.criteria.impl.DNFCriteria; +import com.phonepe.mustang.predicate.impl.IncludedPredicate; + +import lombok.Getter; + +/** + * Search latency for indexes dominated by EQUALITY keys, with and without the equality key lookup. + * + * + */ +public class MustangEqualitySearchBenchmark { + + private static final int REQUEST_POOL_SIZE = 1024; + + @Getter + @State(Scope.Benchmark) + public static class BenchmarkContext { + @Param({ "1000", "10000" }) + private int indexSize; + + @Param({ "DISTINCT_VALUES", "VALUE_LISTS" }) + private String shape; + + @Param({ "true", "false" }) + private boolean equalityKeyLookupEnabled; + + private final ObjectMapper mapper = new ObjectMapper(); + private MustangEngine engine; + private RequestContext[] requests; + + @Setup(Level.Trial) + public void setUp() { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + engine = MustangEngine.builder() + .mapper(mapper) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) + .build(); + final Random random = new Random(7); + final List criterias = new ArrayList<>(indexSize); + final int valuePool = indexSize * 10; + for (int i = 0; i < indexSize; i++) { + final Conjunction.ConjunctionBuilder conjunction = Conjunction.builder(); + if ("DISTINCT_VALUES".equals(shape)) { + conjunction.predicate(IncludedPredicate.builder() + .lhs("$.fenceId") + .values(Sets.newHashSet("FE" + i)) + .build()); + } else { + final Set plans = new HashSet<>(); + while (plans.size() < 50) { + plans.add("PLAN" + random.nextInt(valuePool)); + } + conjunction.predicate(IncludedPredicate.builder() + .lhs("$.pricingPlanId") + .values(plans) + .build()) + .predicate(IncludedPredicate.builder() + .lhs("$.tenant") + .values(Sets.newHashSet("T" + (i % 5))) + .build()); + } + criterias.add(DNFCriteria.builder() + .id("C" + i) + .conjunction(conjunction.build()) + .build()); + } + engine.add(Utils.INDEX_NAME, criterias); + + requests = new RequestContext[REQUEST_POOL_SIZE]; + for (int r = 0; r < REQUEST_POOL_SIZE; r++) { + final Map request = new HashMap<>(); + if ("DISTINCT_VALUES".equals(shape)) { + request.put("fenceId", "FE" + random.nextInt(indexSize * 2)); // ~50% hit rate + } else { + request.put("pricingPlanId", "PLAN" + random.nextInt(valuePool)); + request.put("tenant", "T" + random.nextInt(5)); + } + requests[r] = RequestContext.builder() + .node(mapper.valueToTree(request)) + .build(); + } + } + } + + @State(Scope.Thread) + public static class Cursor { + private int next; + + int next() { + next = (next + 1) & (REQUEST_POOL_SIZE - 1); + return next; + } + } + + @Benchmark + @Fork(value = 1, warmups = 0) + @Warmup(iterations = 2, time = 5) + @Measurement(iterations = 3, time = 5) + @Threads(1) + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void search(final Blackhole blackhole, final BenchmarkContext context, final Cursor cursor) { + blackhole.consume(context.getEngine() + .search(Utils.INDEX_NAME, context.getRequests()[cursor.next()], false)); + } + +} diff --git a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyCNFSearchBenchmark.java b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyCNFSearchBenchmark.java index fb53903..ba5c291 100644 --- a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyCNFSearchBenchmark.java +++ b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyCNFSearchBenchmark.java @@ -56,6 +56,9 @@ public static class BenchmarkContext { @Param({ "10", "100", "1000", "10000" }) private int indexSize; + @Param({ "true", "false" }) + private boolean equalityKeyLookupEnabled; + private final ObjectMapper mapper = new ObjectMapper(); private MustangEngine engine; private RequestContext requestContext; @@ -66,6 +69,7 @@ public void setUp() { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); engine = MustangEngine.builder() .mapper(mapper) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build(); for (int i = 0; i < indexSize; i++) { diff --git a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyDNFSearchBenchmark.java b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyDNFSearchBenchmark.java index 9407947..a8f67c5 100644 --- a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyDNFSearchBenchmark.java +++ b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangOnlyDNFSearchBenchmark.java @@ -56,6 +56,9 @@ public static class BenchmarkContext { @Param({ "10", "100", "1000", "10000" }) private int indexSize; + @Param({ "true", "false" }) + private boolean equalityKeyLookupEnabled; + private final ObjectMapper mapper = new ObjectMapper(); private MustangEngine engine; private RequestContext requestContext; @@ -66,6 +69,7 @@ public void setUp() { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); engine = MustangEngine.builder() .mapper(mapper) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build(); for (int i = 0; i < indexSize; i++) { diff --git a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangSearchBenchmark.java b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangSearchBenchmark.java index fc60e61..c422a48 100644 --- a/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangSearchBenchmark.java +++ b/benchmarking/src/main/java/com/phonepe/mustang/benchmark/benchmarks/MustangSearchBenchmark.java @@ -58,6 +58,9 @@ public static class BenchmarkContext { @Param({ "10", "100", "1000", "10000" }) private int indexSize; + @Param({ "true", "false" }) + private boolean equalityKeyLookupEnabled; + private final ObjectMapper mapper = new ObjectMapper(); private MustangEngine engine; private RequestContext requestContext; @@ -68,6 +71,7 @@ public void setUp() { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); engine = MustangEngine.builder() .mapper(mapper) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build(); for (int i = 0; i < indexSize; i++) { diff --git a/mustang-core/src/main/java/com/phonepe/mustang/MustangEngine.java b/mustang-core/src/main/java/com/phonepe/mustang/MustangEngine.java index ca92953..6182279 100644 --- a/mustang-core/src/main/java/com/phonepe/mustang/MustangEngine.java +++ b/mustang-core/src/main/java/com/phonepe/mustang/MustangEngine.java @@ -53,6 +53,12 @@ public class MustangEngine { private ObjectMapper mapper; @Builder.Default private RankingStrategy rankingStrategy = RankingStrategy.EXPLICIT_WEIGHTS; + /** + * Resolve EQUALITY keys during search via a hash lookup instead of evaluating every key. Search results are + * identical either way; this only changes how matching keys are found. Enabled by default. + */ + @Builder.Default + private boolean equalityKeyLookupEnabled = true; private final IndexingFacade indexingFacde = IndexingFacade.builder() .build(); private final SearchFacade searchFacade = SearchFacade.builder() @@ -107,12 +113,12 @@ public Set search(final String indexName, final RequestContext context) public Set search(final String indexName, final RequestContext context, final int topN) { final Query query = QueryBuilder.buildQuery(context); - return searchFacade.search(indexName, query, topN, true); + return searchFacade.search(indexName, query, topN, true, equalityKeyLookupEnabled); } public Set search(final String indexName, final RequestContext context, final boolean score) { final Query query = QueryBuilder.buildQuery(context); - return searchFacade.search(indexName, query, -1, score); + return searchFacade.search(indexName, query, -1, score, equalityKeyLookupEnabled); } public Set scan(final String indexName, final RequestContext context) { diff --git a/mustang-core/src/main/java/com/phonepe/mustang/search/SearchFacade.java b/mustang-core/src/main/java/com/phonepe/mustang/search/SearchFacade.java index 23a3880..abc014c 100644 --- a/mustang-core/src/main/java/com/phonepe/mustang/search/SearchFacade.java +++ b/mustang-core/src/main/java/com/phonepe/mustang/search/SearchFacade.java @@ -39,10 +39,19 @@ public class SearchFacade { private final IndexingFacade indexingFacade; public Set search(final String indexName, final Query query, final int topN, final boolean score) { + return search(indexName, query, topN, score, true); + } + + public Set search(final String indexName, + final Query query, + final int topN, + final boolean score, + final boolean equalityKeyLookupEnabled) { final Map result = CriteriaSearchHandler.builder() .indexGroup(indexingFacade.getIndexGroup(indexName)) .query(query) .score(score) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build() .handle(); if (!score) { diff --git a/mustang-core/src/main/java/com/phonepe/mustang/search/handler/CriteriaSearchHandler.java b/mustang-core/src/main/java/com/phonepe/mustang/search/handler/CriteriaSearchHandler.java index 63a9d41..23ef51d 100644 --- a/mustang-core/src/main/java/com/phonepe/mustang/search/handler/CriteriaSearchHandler.java +++ b/mustang-core/src/main/java/com/phonepe/mustang/search/handler/CriteriaSearchHandler.java @@ -46,6 +46,8 @@ public class CriteriaSearchHandler implements CriteriaForm.Visitor { @NotNull private final Query query; private final boolean score; + @Builder.Default + private final boolean equalityKeyLookupEnabled = true; private final Map pathValues = Maps.newHashMap(); public Map handle() { @@ -73,6 +75,7 @@ public Matches visitDNF() { .allCriterias(indexGroup.getAllCriterias()) .pathValues(pathValues) .score(score) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build() .getMatches()) .build(); @@ -87,6 +90,7 @@ public Matches visitCNF() { .allCriterias(indexGroup.getAllCriterias()) .pathValues(pathValues) .score(score) + .equalityKeyLookupEnabled(equalityKeyLookupEnabled) .build() .getMatches()) .build(); diff --git a/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/CNFMatcher.java b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/CNFMatcher.java index 19bc4bf..8bf32d7 100644 --- a/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/CNFMatcher.java +++ b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/CNFMatcher.java @@ -22,11 +22,11 @@ import com.phonepe.mustang.index.core.Key; import com.phonepe.mustang.index.core.impl.CNFInvertedIndex; import com.phonepe.mustang.predicate.PredicateType; -import com.phonepe.mustang.preoperation.ChainOperator; import com.phonepe.mustang.search.Query; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -35,7 +35,6 @@ import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -60,6 +59,8 @@ public class CNFMatcher { private final Map allCriterias; private final Map pathValues; private final boolean score; + @Builder.Default + private final boolean equalityKeyLookupEnabled = true; private static DisjunctionPostingEntry getDisjunctionPostingEntry( final TreeMap map, @@ -164,17 +165,14 @@ private Map.Entry>> table, final int k) { final Map> map = table.getOrDefault(k, Collections.emptyMap()); - return getMatchingKeys(map).collect(Collectors.toMap(x -> x, x -> MutablePair.of(0, map.get(x)))) + return getMatchingKeys(k, map).stream() + .collect(Collectors.toMap(x -> x, x -> MutablePair.of(0, map.get(x)))) .entrySet() .toArray(Entry[]::new); } - private Stream getMatchingKeys(final Map> map) { - return map.keySet() - .stream() - .filter(key -> key.getCaveat() - .visit(new CaveatEnforcer(key, - ChainOperator.operate(key.getPreOps(), pathValues.get(key.getName()))))); + private List getMatchingKeys(final int k, final Map> map) { + return MatchingKeyResolver.resolve(invertedIndex, k, map, pathValues, equalityKeyLookupEnabled); } private void initializeCurrentEntriesCNF( diff --git a/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/DNFMatcher.java b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/DNFMatcher.java index 52fe690..5a52b2e 100644 --- a/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/DNFMatcher.java +++ b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/DNFMatcher.java @@ -22,11 +22,11 @@ import com.phonepe.mustang.index.core.Key; import com.phonepe.mustang.index.core.impl.DNFInvertedIndex; import com.phonepe.mustang.predicate.PredicateType; -import com.phonepe.mustang.preoperation.ChainOperator; import com.phonepe.mustang.search.Query; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -35,7 +35,6 @@ import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -55,6 +54,8 @@ public class DNFMatcher { private final Map allCriterias; private final Map pathValues; private final boolean score; + @Builder.Default + private final boolean equalityKeyLookupEnabled = true; private static ConjunctionPostingEntry getConjunctionPostingEntry( final TreeMap map, @@ -138,17 +139,14 @@ private Map.Entry>> table, final int k) { final Map> map = table.getOrDefault(k, Collections.emptyMap()); - return getMatchingKeys(map).collect(Collectors.toMap(x -> x, x -> MutablePair.of(0, map.get(x)))) + return getMatchingKeys(k, map).stream() + .collect(Collectors.toMap(x -> x, x -> MutablePair.of(0, map.get(x)))) .entrySet() .toArray(Entry[]::new); } - private Stream getMatchingKeys(final Map> map) { - return map.keySet() - .stream() - .filter(key -> key.getCaveat() - .visit(new CaveatEnforcer(key, - ChainOperator.operate(key.getPreOps(), pathValues.get(key.getName()))))); + private List getMatchingKeys(final int k, final Map> map) { + return MatchingKeyResolver.resolve(invertedIndex, k, map, pathValues, equalityKeyLookupEnabled); } private void initializeCurrentEntriesDNF( diff --git a/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/MatchingKeyResolver.java b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/MatchingKeyResolver.java new file mode 100644 index 0000000..adbebd9 --- /dev/null +++ b/mustang-core/src/main/java/com/phonepe/mustang/search/matcher/MatchingKeyResolver.java @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2022 PhonePe India Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.phonepe.mustang.search.matcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.phonepe.mustang.index.core.EqualityKeyLookup; +import com.phonepe.mustang.index.core.InvertedIndex; +import com.phonepe.mustang.index.core.Key; +import com.phonepe.mustang.preoperation.ChainOperator; + +import lombok.experimental.UtilityClass; + +/** + * Finds the keys of a posting-list map that are satisfied by the current request. + * + *

+ * With the equality lookup enabled, {@code EQUALITY} keys are resolved through a hash lookup per + * {@code (path, preOperations)} group, and only the remaining keys are evaluated one by one. With it disabled, every + * key is evaluated one by one (the original behaviour). Both modes return the same set of keys. + *

+ */ +@UtilityClass +class MatchingKeyResolver { + + static List resolve(final InvertedIndex invertedIndex, + final int size, + final Map postingLists, + final Map pathValues, + final boolean equalityKeyLookupEnabled) { + if (!equalityKeyLookupEnabled) { + return scan(postingLists.keySet(), pathValues); + } + final EqualityKeyLookup lookup = invertedIndex.getEqualityKeyLookup(size, postingLists); + final List matchingKeys = scan(lookup.getScanKeys(), pathValues); + lookup.getEqualityGroups() + .forEach((group, buckets) -> { + final Object lhsValue = ChainOperator.operate(group.getPreOps(), pathValues.get(group.getName())); + final Object normalised = EqualityKeyLookup.normaliseRequestValue(lhsValue); + if (normalised == EqualityKeyLookup.NO_MATCH) { + return; + } + if (normalised == EqualityKeyLookup.FALLBACK) { + buckets.values() + .forEach(keys -> keys.stream() + .filter(key -> satisfies(key, lhsValue)) + .forEach(matchingKeys::add)); + return; + } + final List keys = buckets.get(normalised); + if (Objects.nonNull(keys)) { + matchingKeys.addAll(keys); + } + }); + return matchingKeys; + } + + private static List scan(final Iterable keys, final Map pathValues) { + final List matchingKeys = new ArrayList<>(); + keys.forEach(key -> { + if (satisfies(key, ChainOperator.operate(key.getPreOps(), pathValues.get(key.getName())))) { + matchingKeys.add(key); + } + }); + return matchingKeys; + } + + private static boolean satisfies(final Key key, final Object lhsValue) { + return key.getCaveat() + .visit(new CaveatEnforcer(key, lhsValue)); + } +} diff --git a/mustang-core/src/test/java/com/phonepe/mustang/search/EqualityKeyLookupSearchTest.java b/mustang-core/src/test/java/com/phonepe/mustang/search/EqualityKeyLookupSearchTest.java new file mode 100644 index 0000000..1653ef8 --- /dev/null +++ b/mustang-core/src/test/java/com/phonepe/mustang/search/EqualityKeyLookupSearchTest.java @@ -0,0 +1,380 @@ +/** + * Copyright (c) 2022 PhonePe India Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.phonepe.mustang.search; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import com.phonepe.mustang.MustangEngine; +import com.phonepe.mustang.common.RequestContext; +import com.phonepe.mustang.composition.impl.Conjunction; +import com.phonepe.mustang.composition.impl.Disjunction; +import com.phonepe.mustang.criteria.Criteria; +import com.phonepe.mustang.criteria.impl.CNFCriteria; +import com.phonepe.mustang.criteria.impl.DNFCriteria; +import com.phonepe.mustang.detail.impl.RangeDetail; +import com.phonepe.mustang.detail.impl.RegexDetail; +import com.phonepe.mustang.predicate.Predicate; +import com.phonepe.mustang.predicate.impl.ExcludedPredicate; +import com.phonepe.mustang.predicate.impl.IncludedPredicate; +import com.phonepe.mustang.preoperation.impl.AdditionPreOperation; +import com.phonepe.mustang.preoperation.impl.ModuloPreOperation; +import com.phonepe.mustang.preoperation.impl.SizePreOperation; +import com.phonepe.mustang.ratify.RatificationResult; + +/** + * Verifies that resolving EQUALITY keys through the hash lookup returns exactly the same search results as evaluating + * every key (the original behaviour). + */ +public class EqualityKeyLookupSearchTest { + + private static final String INDEX = "test"; + private final ObjectMapper mapper = new ObjectMapper(); + private MustangEngine lookupEngine; + private MustangEngine scanEngine; + + @Before + public void setUp() { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + lookupEngine = MustangEngine.builder() + .mapper(mapper) + .build(); + scanEngine = MustangEngine.builder() + .mapper(mapper) + .equalityKeyLookupEnabled(false) + .build(); + } + + @Test + public void testNumericValuesCompareByDoubleValue() { + add(dnf("INT", included("$.n", 5))); + add(dnf("LONG", included("$.n", 7L))); + add(dnf("DOUBLE", included("$.n", 2.5))); + add(dnf("DECIMAL", included("$.n", new BigDecimal("9.0")))); + add(dnf("NEG_ZERO", included("$.n", -0.0d))); + add(dnf("STRING_FIVE", included("$.n", "5"))); + + assertResults(request("n", 5), "INT"); + assertResults(request("n", 5.0), "INT"); + assertResults(request("n", 7), "LONG"); + assertResults(request("n", 2.5), "DOUBLE"); + assertResults(request("n", 9), "DECIMAL"); + assertResults(request("n", 0), "NEG_ZERO"); + assertResults(request("n", "5"), "STRING_FIVE"); + assertResults(request("n", 6)); + } + + @Test + public void testBooleansAndStringsDoNotCrossMatch() { + add(dnf("BOOL", included("$.b", true))); + add(dnf("STRING_TRUE", included("$.b", "true"))); + + assertResults(request("b", true), "BOOL"); + assertResults(request("b", "true"), "STRING_TRUE"); + assertResults(request("b", false)); + } + + @Test + public void testListRequestValueUsesFirstElement() { + add(dnf("X", included("$.ids", "X"))); + add(dnf("Y", included("$.ids", "Y"))); + + assertResults(request("ids", Arrays.asList("X", "Y")), "X"); + assertResults(request("ids", Arrays.asList("Y", "X")), "Y"); + } + + @Test + public void testMissingAndObjectValuesDoNotMatch() { + add(dnf("A", included("$.a", "A1"))); + + assertResults(request("other", "A1")); + final Map nested = new HashMap<>(); + nested.put("x", "A1"); + assertResults(request("a", nested)); + } + + @Test + public void testLargeValueListPredicate() { + final Set plans = Sets.newHashSet(); + for (int i = 0; i < 200; i++) { + plans.add("BPP" + i); + } + add(dnf("PLANS", IncludedPredicate.builder() + .lhs("$.pricingPlanId") + .values(plans) + .build())); + add(dnf("OTHER", included("$.pricingPlanId", "MPP1"))); + + assertResults(request("pricingPlanId", "BPP137"), "PLANS"); + assertResults(request("pricingPlanId", "MPP1"), "OTHER"); + assertResults(request("pricingPlanId", "BPP999")); + } + + @Test + public void testPreOperationsAreAppliedPerGroup() { + add(dnf("MOD", IncludedPredicate.builder() + .lhs("$.id") + .preOperation(ModuloPreOperation.builder() + .rhs(100) + .build()) + .values(Sets.newHashSet(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)) + .build())); + add(dnf("MOD_PLUS_ONE", IncludedPredicate.builder() + .lhs("$.id") + .preOperations(Arrays.asList(ModuloPreOperation.builder() + .rhs(100) + .build(), + AdditionPreOperation.builder() + .rhs(1) + .build())) + .values(Sets.newHashSet(8)) + .build())); + add(dnf("IDENTITY", included("$.id", 12307))); + add(dnf("SIZE", IncludedPredicate.builder() + .lhs("$.items") + .preOperation(SizePreOperation.builder() + .build()) + .values(Sets.newHashSet(3)) + .build())); + + assertResults(request("id", 12307), "MOD", "MOD_PLUS_ONE", "IDENTITY"); + assertResults(request("id", 12350)); + assertResults(request("items", Arrays.asList("a", "b", "c")), "SIZE"); + } + + @Test + public void testExcludedEqualityAndMixedCaveats() { + add(dnf("MIXED", included("$.state", "MH"), ExcludedPredicate.builder() + .lhs("$.city") + .values(Sets.newHashSet("PUNE")) + .build(), + IncludedPredicate.builder() + .lhs("$.amount") + .detail(RangeDetail.builder() + .lowerBound(500) + .includeLowerBound(true) + .upperBound(2000) + .includeUpperBound(true) + .build()) + .build(), + IncludedPredicate.builder() + .lhs("$.user") + .detail(RegexDetail.builder() + .regex(".*0[0-9]$") + .build()) + .build())); + + final Map request = new HashMap<>(); + request.put("state", "MH"); + request.put("city", "MUMBAI"); + request.put("amount", 1500); + request.put("user", "U105"); + assertResults(request, "MIXED"); + + request.put("city", "PUNE"); + assertResults(request); + + request.put("city", "MUMBAI"); + request.put("amount", 2500); + assertResults(request); + } + + @Test + public void testCnfCriteria() { + add(CNFCriteria.builder() + .id("CNF") + .disjunction(Disjunction.builder() + .predicate(included("$.pincode", "560034")) + .predicate(included("$.city", "BLR")) + .build()) + .disjunction(Disjunction.builder() + .predicate(ExcludedPredicate.builder() + .lhs("$.state") + .values(Sets.newHashSet("TN")) + .build()) + .build()) + .build()); + + assertResults(requestOf("pincode", "560034", "state", "KA"), "CNF"); + assertResults(requestOf("city", "BLR", "state", "KA"), "CNF"); + assertResults(requestOf("city", "BLR", "state", "TN")); + assertResults(requestOf("city", "MUM", "state", "KA")); + } + + @Test + public void testLookupIsRebuiltAfterIndexChanges() { + final Criteria v1 = dnf("FENCE_OFFER", included("$.fenceId", "FE1")); + add(v1); + assertResults(request("fenceId", "FE1"), "FENCE_OFFER"); + + final Criteria v2 = dnf("FENCE_OFFER", included("$.fenceId", "FE2")); + lookupEngine.update(INDEX, v2); + scanEngine.update(INDEX, v2); + assertResults(request("fenceId", "FE1")); + assertResults(request("fenceId", "FE2"), "FENCE_OFFER"); + + add(dnf("NEW_OFFER", included("$.fenceId", "FE3"))); + assertResults(request("fenceId", "FE3"), "NEW_OFFER"); + + lookupEngine.delete(INDEX, v2); + scanEngine.delete(INDEX, v2); + assertResults(request("fenceId", "FE2")); + } + + @Test + public void testManyFenceIdsOnOnePath() { + final List criterias = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + criterias.add(dnf("OFFER_" + i, included("$.fenceId", "FE" + i))); + } + lookupEngine.add(INDEX, criterias); + scanEngine.add(INDEX, criterias); + assertResults(request("fenceId", "FE432"), "OFFER_432"); + assertResults(request("fenceId", "FE1000")); + } + + @Test + public void testRandomisedParityWithScan() { + final Random random = new Random(42); + final String[] paths = { "$.a", "$.b", "$.c", "$.d", "$.e" }; + for (int i = 0; i < 400; i++) { + final int predicates = 1 + random.nextInt(3); + final List list = new ArrayList<>(); + for (int p = 0; p < predicates; p++) { + final String path = paths[random.nextInt(paths.length)]; + final Set values = Sets.newHashSet(randomValue(random), randomValue(random)); + list.add(random.nextInt(5) == 0 ? ExcludedPredicate.builder() + .lhs(path) + .values(values) + .build() + : IncludedPredicate.builder() + .lhs(path) + .values(values) + .build()); + } + if (random.nextBoolean()) { + add(dnf("D" + i, list.toArray(new Predicate[0]))); + } else { + final CNFCriteria.CNFCriteriaBuilder builder = CNFCriteria.builder() + .id("C" + i); + list.forEach(predicate -> builder.disjunction(Disjunction.builder() + .predicate(predicate) + .build())); + add(builder.build()); + } + } + for (int r = 0; r < 1000; r++) { + final Map request = new HashMap<>(); + for (final String path : paths) { + if (random.nextInt(4) != 0) { + request.put(path.substring(2), randomValue(random)); + } + } + assertSameResults(request); + } + } + + private Object randomValue(final Random random) { + final int v = random.nextInt(6); + switch (random.nextInt(4)) { + case 0: + return v; + case 1: + return (double) v; + case 2: + return "V" + v; + default: + return v % 2 == 0; + } + } + + private void add(final Criteria criteria) { + lookupEngine.add(INDEX, criteria); + scanEngine.add(INDEX, criteria); + } + + private static Criteria dnf(final String id, final Predicate... predicates) { + final Conjunction.ConjunctionBuilder conjunction = Conjunction.builder(); + Arrays.stream(predicates) + .forEach(conjunction::predicate); + return DNFCriteria.builder() + .id(id) + .conjunction(conjunction.build()) + .build(); + } + + private static Predicate included(final String path, final Object value) { + return IncludedPredicate.builder() + .lhs(path) + .values(Sets.newHashSet(value)) + .build(); + } + + private static Map request(final String key, final Object value) { + final Map request = new HashMap<>(); + request.put(key, value); + return request; + } + + private static Map requestOf(final String k1, final Object v1, final String k2, final Object v2) { + final Map request = request(k1, v1); + request.put(k2, v2); + return request; + } + + private Set assertSameResults(final Map request) { + final RequestContext context = RequestContext.builder() + .node(mapper.valueToTree(request)) + .build(); + final Set lookupResults = lookupEngine.search(INDEX, context); + final Set scanResults = scanEngine.search(INDEX, context); + assertThat("Mismatch for request " + request, lookupResults, is(scanResults)); + return lookupResults; + } + + private void assertResults(final Map request, final String... expected) { + final Set results = assertSameResults(request); + if (expected.length == 0) { + assertThat(results, is(empty())); + } else { + assertThat(results, containsInAnyOrder(expected)); + } + } + + private void assertRatified() { + lookupEngine.ratify(INDEX); + final RatificationResult result = lookupEngine.getRatificationResult(INDEX); + assertThat(result.getStatus(), is(true)); + assertThat(result.getAnamolyDetails(), is(empty())); + } +} diff --git a/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java b/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java new file mode 100644 index 0000000..e12a3de --- /dev/null +++ b/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java @@ -0,0 +1,161 @@ +/** + * Copyright (c) 2022 PhonePe India Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.phonepe.mustang.index.core; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.phonepe.mustang.detail.Caveat; +import com.phonepe.mustang.preoperation.PreOperation; + +import lombok.Getter; +import lombok.Value; + +/** + * Read-only lookup over the keys of one posting-list map (one entry of {@link InvertedIndex#getTable()}). + * + *

+ * {@link Caveat#EQUALITY} keys whose value has well defined equality semantics (strings, booleans and standard + * numbers) are bucketed by {@code (path, preOperations)} and then by a normalised value, so that the keys + * matching a request can be found with a hash lookup instead of evaluating every key. All other keys (other caveats, + * or equality keys with any other value type) are kept in {@link #getScanKeys()} and must be evaluated one by one, + * exactly as before. + *

+ * + *

+ * Normalisation mirrors {@code Utils.compare}: numbers are compared by {@code doubleValue()}, booleans only match + * booleans, strings only match strings, and a collection request value is compared using its first element only. + *

+ * + *

+ * The lookup is a pure function of the source map. The source map is never mutated by the indexers (they always + * install a new map instance), so a lookup stays valid as long as {@link #getSource()} is the current map. + *

+ */ +@Getter +public final class EqualityKeyLookup { + + /** Marker returned by {@link #normaliseRequestValue(Object)} when no hashed key can match. */ + public static final Object NO_MATCH = new Object(); + + /** + * Marker returned by {@link #normaliseRequestValue(Object)} when the value cannot be normalised safely, and the + * keys of the group must be evaluated with the regular caveat check. + */ + public static final Object FALLBACK = new Object(); + + private final Map source; + private final Map>> equalityGroups; + private final List scanKeys; + + private EqualityKeyLookup(final Map source, + final Map>> equalityGroups, + final List scanKeys) { + this.source = source; + this.equalityGroups = equalityGroups; + this.scanKeys = scanKeys; + } + + public static EqualityKeyLookup of(final Map source) { + final Map>> groups = new HashMap<>(); + final List scan = new ArrayList<>(); + source.keySet() + .forEach(key -> { + final Object normalisedValue = Caveat.EQUALITY.equals(key.getCaveat()) + ? normaliseKeyValue(key.getValue()) + : null; + if (Objects.isNull(normalisedValue)) { + scan.add(key); + return; + } + groups.computeIfAbsent(new KeyGroup(key.getName(), key.getPreOps()), x -> new HashMap<>()) + .computeIfAbsent(normalisedValue, x -> new ArrayList<>(1)) + .add(key); + }); + return new EqualityKeyLookup(source, Collections.unmodifiableMap(groups), Collections.unmodifiableList(scan)); + } + + /** + * @return the normalised hash value of an equality key, or {@code null} if the key must be scanned. + */ + public static Object normaliseKeyValue(final Object value) { + if (value instanceof String || value instanceof Boolean) { + return value; + } + if (isStandardNumber(value)) { + return normaliseNumber((Number) value); + } + return null; + } + + /** + * Normalises a (pre-operated) request value the same way {@code Utils.compare} would compare it. + * + * @return the hash value to look up, {@link #NO_MATCH} if no hashed key can match, or {@link #FALLBACK} if the + * value has to be checked with the regular caveat evaluation. + */ + public static Object normaliseRequestValue(final Object lhsValue) { + if (Objects.isNull(lhsValue)) { + return NO_MATCH; + } + Object value = lhsValue; + if (value instanceof Collection) { + if (!(value instanceof List) || ((List) value).isEmpty()) { + return FALLBACK; + } + value = ((List) value).get(0); + if (Objects.isNull(value)) { + return FALLBACK; + } + } + if (value instanceof String || value instanceof Boolean) { + return value; + } + if (value instanceof Number) { + final Object normalised = normaliseNumber((Number) value); + return Objects.isNull(normalised) ? NO_MATCH : normalised; + } + // Hashed keys are strings, booleans or numbers; none of them equal any other type. + return NO_MATCH; + } + + private static Object normaliseNumber(final Number number) { + final double value = number.doubleValue(); + if (Double.isNaN(value)) { + return null; // NaN never equals anything under doubleValue() comparison + } + return value == 0.0d ? 0.0d : value; // fold -0.0 into 0.0, as == does + } + + private static boolean isStandardNumber(final Object value) { + return value instanceof Integer || value instanceof Long || value instanceof Double + || value instanceof Float || value instanceof Short || value instanceof Byte + || value instanceof BigDecimal || value instanceof BigInteger; + } + + @Value + public static class KeyGroup { + String name; + List preOps; + } +} diff --git a/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java b/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java index 02d7c46..51c74b3 100644 --- a/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java +++ b/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java @@ -18,6 +18,7 @@ import java.util.Comparator; import java.util.Map; +import java.util.Objects; import java.util.PriorityQueue; import java.util.TreeMap; import java.util.TreeSet; @@ -25,11 +26,17 @@ import javax.validation.constraints.NotNull; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Maps; import com.phonepe.mustang.criteria.CriteriaForm; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; @Data @AllArgsConstructor @@ -40,6 +47,33 @@ public abstract class InvertedIndex { private final Map>> table = Maps.newConcurrentMap(); private final Map> idCache = Maps.newConcurrentMap(); private final Map> linkages = Maps.newConcurrentMap(); + /** + * Derived, per conjunction/disjunction size lookups over {@link #table}. Never serialised; rebuilt lazily + * whenever the posting-list map for a size is replaced by the indexers. + */ + @JsonIgnore + @Getter(AccessLevel.NONE) + @Setter(AccessLevel.NONE) + @EqualsAndHashCode.Exclude + @ToString.Exclude + private final transient Map equalityKeyLookups = Maps.newConcurrentMap(); + + /** + * Returns the equality key lookup for the given size, rebuilding it if the posting-list map has been replaced + * since it was last built. + * + * @param size conjunction (DNF) or disjunction (CNF) size, i.e. the key of {@link #table} + * @param postingLists the current posting-list map for that size + */ + public EqualityKeyLookup getEqualityKeyLookup(final int size, final Map postingLists) { + final EqualityKeyLookup existing = equalityKeyLookups.get(size); + if (Objects.nonNull(existing) && existing.getSource() == postingLists) { + return existing; + } + final EqualityKeyLookup rebuilt = EqualityKeyLookup.of(postingLists); + equalityKeyLookups.put(size, rebuilt); + return rebuilt; + } public Integer getInternalIdFromCache(final String externalId) { return idCache.computeIfAbsent(externalId, x -> { From 8035d9981f5addb2f4976efc8959f737646c9fd4 Mon Sep 17 00:00:00 2001 From: Anant Sahrawat Date: Fri, 30 Oct 2026 15:26:45 +0530 Subject: [PATCH 2/2] Fix javadoc references to Lombok-generated members --- .../com/phonepe/mustang/index/core/EqualityKeyLookup.java | 6 +++--- .../java/com/phonepe/mustang/index/core/InvertedIndex.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java b/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java index e12a3de..948b18c 100644 --- a/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java +++ b/mustang-models/src/main/java/com/phonepe/mustang/index/core/EqualityKeyLookup.java @@ -32,13 +32,13 @@ import lombok.Value; /** - * Read-only lookup over the keys of one posting-list map (one entry of {@link InvertedIndex#getTable()}). + * Read-only lookup over the keys of one posting-list map (one entry of {@code InvertedIndex#getTable()}). * *

* {@link Caveat#EQUALITY} keys whose value has well defined equality semantics (strings, booleans and standard * numbers) are bucketed by {@code (path, preOperations)} and then by a normalised value, so that the keys * matching a request can be found with a hash lookup instead of evaluating every key. All other keys (other caveats, - * or equality keys with any other value type) are kept in {@link #getScanKeys()} and must be evaluated one by one, + * or equality keys with any other value type) are kept in {@code getScanKeys()} and must be evaluated one by one, * exactly as before. *

* @@ -49,7 +49,7 @@ * *

* The lookup is a pure function of the source map. The source map is never mutated by the indexers (they always - * install a new map instance), so a lookup stays valid as long as {@link #getSource()} is the current map. + * install a new map instance), so a lookup stays valid as long as {@code getSource()} is the current map. *

*/ @Getter diff --git a/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java b/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java index 51c74b3..6521c89 100644 --- a/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java +++ b/mustang-models/src/main/java/com/phonepe/mustang/index/core/InvertedIndex.java @@ -48,7 +48,7 @@ public abstract class InvertedIndex { private final Map> idCache = Maps.newConcurrentMap(); private final Map> linkages = Maps.newConcurrentMap(); /** - * Derived, per conjunction/disjunction size lookups over {@link #table}. Never serialised; rebuilt lazily + * Derived, per conjunction/disjunction size lookups over {@code table}. Never serialised; rebuilt lazily * whenever the posting-list map for a size is replaced by the indexers. */ @JsonIgnore @@ -62,7 +62,7 @@ public abstract class InvertedIndex { * Returns the equality key lookup for the given size, rebuilding it if the posting-list map has been replaced * since it was last built. * - * @param size conjunction (DNF) or disjunction (CNF) size, i.e. the key of {@link #table} + * @param size conjunction (DNF) or disjunction (CNF) size, i.e. the key of {@code table} * @param postingLists the current posting-list map for that size */ public EqualityKeyLookup getEqualityKeyLookup(final int size, final Map postingLists) {