diff --git a/changelog/unreleased/auxindexjoin-qparser.yml b/changelog/unreleased/auxindexjoin-qparser.yml
new file mode 100644
index 000000000000..3548e368da76
--- /dev/null
+++ b/changelog/unreleased/auxindexjoin-qparser.yml
@@ -0,0 +1,10 @@
+title: >
+ Introducing {!auxIndexJoin} query for query-time join with auxiliary index.
+type: added
+authors:
+ - name: Mikhail Khludnev
+ nick: mkhl
+links:
+ - name: SOLR-18307
+ url: https://issues.apache.org/jira/browse/SOLR-18307
+
diff --git a/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java
new file mode 100644
index 000000000000..928e2f0dde5b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CloseHook;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexJoinConfig;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager;
+import org.apache.solr.util.RefCounted;
+import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Query parser exercising {@link AuxIndexManager} inside a {@link SolrCore}: it mimics {@link
+ * ScoreJoinQParserPlugin}'s local parameters, but resolves matches through the sidecar join index
+ * instead of {@link org.apache.lucene.search.join.JoinUtil}. Local parameters:
+ *
+ *
+ * - from - "foreign key" field name, collected while enumerating the subordinate query (the
+ * local parameter value).
+ *
- fromIndex - optional core name to run the subordinate query against, when it differs from
+ * this core; cross-core joins are the reason {@link AuxIndexManager} exists in the first
+ * place, so this mirrors {@link ScoreJoinQParserPlugin}'s
fromIndex, including
+ * SolrCloud alias/collection resolution via {@link ScoreJoinQParserPlugin#getCoreName}.
+ * - to - "primary key" field name looked up in this core's index.
+ *
+ *
+ * Example: {@code q={!aijoin from=manu_id_s to=id fromIndex=products}foo}.
+ *
+ * Unlike {@link ScoreJoinQParserPlugin.OtherCoreJoinQuery}, which only borrows the from-side
+ * searcher long enough to build a self-contained {@code Query} in {@code createWeight}, an {@link
+ * org.apache.solr.search.join.auxindexjoin.AuxIndexJoinQuery} keeps reading the from-side searcher
+ * on every {@code scorerSupplier} call (it may lazily build missing pair columns per to-segment),
+ * so a cross-core from-searcher is pinned open for the whole request via {@link
+ * SolrRequestInfo#addCloseHook}, the same mechanism {@link
+ * org.apache.solr.search.JoinQuery.JoinQueryWeight} uses for the regular {@code {!join}}.
+ *
+ *
One {@link AuxIndexManager} is opened per core in {@link #inform(SolrCore)}, backed by a
+ * directory under the core's dataDir (configurable via the {@code dir} init parameter, resolved
+ * relative to dataDir unless absolute), and closed when the core closes. The remaining init
+ * parameters mirror {@link AuxIndexJoinConfig}: {@code singleFieldPerSegment}, {@code
+ * blockingRefresh}, {@code useFromSideThreads}, and {@code sweepSamplingInterval} (seconds). This
+ * sidecar always belongs to the "to" side core -- the one this plugin is registered in.
+ *
+ *
Why this implements {@link QueryResponseWriter}: {@link
+ * org.apache.solr.core.SolrResourceLoader}'s {@code awareCompatibility} allowlist (see SOLR-8311)
+ * only lets specific plugin base types implement {@link SolrCoreAware}, and {@code QParserPlugin}
+ * isn't one of them, so a plain {@code implements SolrCoreAware} fails core load with "Invalid
+ * 'Aware' object". {@code QueryResponseWriter} is on the allowlist and happens to be the cheapest
+ * interface there to satisfy (two abstract methods, both unreachable stubs below -- this class is
+ * never registered as a {@code }). This is safe here specifically because
+ * {@code QParserPlugin} instances are loaded once per core load/reload via {@link
+ * org.apache.solr.core.PluginBag}, exactly like the already-whitelisted {@link
+ * org.apache.solr.handler.component.SearchComponent} -- never created ad-hoc per request ({@link
+ * QParser#getParser(String, SolrQueryRequest)} resolves the already registered instance via {@code
+ * req.getCore().getQueryPlugin(name)}).
+ */
+public class AuxIndexJoinQParserPlugin extends QParserPlugin
+ implements QueryResponseWriter, SolrCoreAware {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Init parameter: directory holding the sidecar join index, resolved against the core's dataDir
+ * unless absolute. Defaults to {@value #DEFAULT_DIR}.
+ */
+ public static final String DIR = "dir";
+
+ public static final String NAME = "auxIndexJoin";
+
+ public static final String DEFAULT_DIR = "aux-index-join";
+
+ /** Init parameter: whether each pair column is flushed into its own sidecar segment. */
+ public static final String SINGLE_FIELD_PER_SEGMENT = "singleFieldPerSegment";
+
+ /** Init parameter: whether writing a batch of pair columns blocks until it is searchable. */
+ public static final String BLOCKING_REFRESH = "blockingRefresh";
+
+ /**
+ * Init parameter: whether loading from-side leaves that feed a join build is parallelized across
+ * executor threads. Defaults to {@code true}.
+ */
+ public static final String USE_FROM_SIDE_THREADS = "useFromSideThreads";
+
+ /**
+ * Init parameter: how often (in seconds) {@code AuxIndexManager.onCreateWeight} samples searcher
+ * state for the dead-pair reaper. Non-positive means sample on every call. Defaults to 60.
+ */
+ public static final String SWEEP_SAMPLING_INTERVAL = "sweepSamplingInterval";
+
+ private String configuredDir = DEFAULT_DIR;
+ private final AuxIndexJoinConfig joinIndexConfig = new AuxIndexJoinConfig();
+
+ private volatile AuxIndexManager joinIndex;
+
+ @Override
+ public void init(NamedList args) {
+ super.init(args);
+ if (args != null) {
+ SolrParams params = args.toSolrParams();
+ configuredDir = params.get(DIR, DEFAULT_DIR);
+ joinIndexConfig.setSingleFieldPerSegment(
+ params.getBool(SINGLE_FIELD_PER_SEGMENT, joinIndexConfig.getSingleFieldPerSegment()));
+ joinIndexConfig.setBlockingRefresh(
+ params.getBool(BLOCKING_REFRESH, joinIndexConfig.getBlockingRefresh()));
+ joinIndexConfig.setUseFromSideThreads(
+ params.getBool(USE_FROM_SIDE_THREADS, joinIndexConfig.getUseFromSideThreads()));
+ joinIndexConfig.setSweepSamplingInterval(
+ params.getLong(SWEEP_SAMPLING_INTERVAL, 60), TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public void inform(SolrCore core) {
+ Path path = Path.of(configuredDir);
+ if (!path.isAbsolute()) {
+ path = Path.of(core.getDataDir()).resolve(path);
+ } else {
+ core.getCoreContainer().assertPathAllowed(path);
+ }
+ Directory directory = null;
+ try {
+ directory =
+ core.getDirectoryFactory()
+ .get(path.toString(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ AuxIndexJoinConfig config = joinIndexConfig;
+ joinIndex = new AuxIndexManager(directory, config);
+ } catch (IOException | RuntimeException e) {
+ if (directory != null) {
+ try {
+ core.getDirectoryFactory().release(directory);
+ } catch (IOException releaseException) {
+ e.addSuppressed(releaseException);
+ }
+ }
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Failed to open AuxIndexManager at " + path, e);
+ }
+ final Directory capturedDirectory = directory;
+ core.addCloseHook(
+ new CloseHook() {
+ @Override
+ public void preClose(SolrCore core) {
+ try {
+ joinIndex.close();
+ } catch (IOException e) {
+ log.warn("Failed closing AuxIndexManager", e);
+ } finally {
+ try {
+ core.getDirectoryFactory().release(capturedDirectory);
+ } catch (IOException e) {
+ log.warn("Failed releasing AuxIndexManager directory {}", capturedDirectory, e);
+ }
+ }
+ }
+ });
+ }
+
+ // QueryResponseWriter stubs, unreachable: implemented only to satisfy SolrCoreAware's allowlist,
+ // see the class javadoc. This plugin is never registered as a .
+
+ @Override
+ public void write(
+ OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType) {
+ throw new UnsupportedOperationException(
+ AuxIndexJoinQParserPlugin.class.getSimpleName()
+ + " is a QParserPlugin, not a QueryResponseWriter");
+ }
+
+ @Override
+ public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
+ throw new UnsupportedOperationException(
+ AuxIndexJoinQParserPlugin.class.getSimpleName()
+ + " is a QParserPlugin, not a QueryResponseWriter");
+ }
+
+ @Override
+ public QParser createParser(
+ String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
+ return new QParser(qstr, localParams, params, req) {
+ @Override
+ public Query parse() throws SyntaxError {
+ if (joinIndex == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "AuxIndexJoinQParserPlugin is not initialized; is it registered as a ?");
+ }
+ final String fromField = getParam("from");
+ final String toField = getParam("to");
+ if (fromField == null || toField == null) {
+ throw new SyntaxError("auxIndexJoin query parser requires 'from' and 'to' local params");
+ }
+ final String fromIndex = localParams.get("fromIndex");
+ final String v = localParams.get(CommonParams.VALUE);
+ final String myCore = req.getCore().getCoreDescriptor().getName();
+
+ final Query fromQuery;
+ final IndexSearcher fromSearcher;
+ ExecutorService fromExecutor;
+ if (fromIndex != null && !fromIndex.equals(myCore)) {
+ CoreContainer container = req.getCoreContainer();
+ String coreName =
+ ScoreJoinQParserPlugin.getCoreName(
+ fromIndex, container, req.getCore(), toField, fromField, localParams);
+ SolrCore fromCore = container.getCore(coreName);
+ if (fromCore == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + coreName);
+ }
+ SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+ if (info == null) {
+ fromCore.close();
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Cross-core auxIndexJoin must have SolrRequestInfo");
+ }
+ // released once this request completes: the from-side searcher is read on every
+ // scorerSupplier() call, not just while building this query, so it must outlive parse()
+ info.addCloseHook(fromCore);
+ try (SolrQueryRequestBase otherReq = new SolrQueryRequestBase(fromCore, params)) {
+ fromQuery = QParser.getParser(v, otherReq).getQuery();
+ }
+ RefCounted fromRef = fromCore.getSearcher(false, true, null);
+ info.addCloseHook(fromRef::decref);
+ fromSearcher = fromRef.get();
+ fromExecutor = (ExecutorService) fromCore.getCoreContainer().getIndexSearcherExecutor();
+ } else {
+ fromQuery = subQuery(v, null).getQuery();
+ fromSearcher = req.getSearcher();
+ fromExecutor = (ExecutorService) req.getCoreContainer().getIndexSearcherExecutor();
+ }
+
+ return joinIndex.newJoinQuery(fromField, fromQuery, fromSearcher, toField, fromExecutor);
+ }
+ };
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java
new file mode 100644
index 000000000000..779e5aac6d1a
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Holds the configuration used to create an {@link AuxIndexManager}. Every setter returns {@link
+ * AuxIndexJoinConfig} to allow chaining settings conveniently, for example:
+ *
+ *
+ * AuxIndexJoinConfig config = new AuxIndexJoinConfig().setBlockingRefresh(false);
+ * AuxIndexManager joinIndex = new AuxIndexManager(joinDir, config);
+ *
+ *
+ * Once passed to {@link AuxIndexManager#AuxIndexManager(Directory, AuxIndexJoinConfig)}, changes
+ * to this object no longer affect the created {@link AuxIndexManager} instance.
+ */
+public final class AuxIndexJoinConfig {
+
+ private boolean singleFieldPerSegment = false;
+ private boolean blockingRefresh = true;
+ private boolean useFromSideThreads = true;
+ private long sweepSamplingIntervalNanos = TimeUnit.MINUTES.toNanos(1);
+
+ /** Sole constructor, using the default settings documented on each setter. */
+ public AuxIndexJoinConfig() {}
+
+ /**
+ * Whether each pair column is flushed into its own sidecar segment, rather than batching every
+ * pair column built in the same round into one segment. Default is {@code false}: many columns
+ * per segment, traded off against a longer sweep to reclaim any that become dead.
+ */
+ public AuxIndexJoinConfig setSingleFieldPerSegment(boolean singleFieldPerSegment) {
+ this.singleFieldPerSegment = singleFieldPerSegment;
+ return this;
+ }
+
+ /** Returns the current value set via {@link #setSingleFieldPerSegment}. */
+ public boolean getSingleFieldPerSegment() {
+ return singleFieldPerSegment;
+ }
+
+ /**
+ * Whether writing a batch of pair columns blocks until the sidecar's {@link
+ * org.apache.lucene.search.SearcherManager} is refreshed past it, so the freshly built pairs are
+ * visible to the caller that triggered the build. Default is {@code true}.
+ */
+ public AuxIndexJoinConfig setBlockingRefresh(boolean blockingRefresh) {
+ this.blockingRefresh = blockingRefresh;
+ return this;
+ }
+
+ /** Returns the current value set via {@link #setBlockingRefresh}. */
+ public boolean getBlockingRefresh() {
+ return blockingRefresh;
+ }
+
+ /**
+ * Whether loading the from-side leaves that feed a join build is parallelized across the caller's
+ * executor threads. When {@code true} (the default) each from-side segment is loaded on a
+ * separate executor thread; when {@code false} they are loaded sequentially on the calling
+ * thread. Set to {@code false} to bound the load to a single thread, e.g. when the executor is
+ * contended or to keep queries deterministic.
+ */
+ public AuxIndexJoinConfig setUseFromSideThreads(boolean useFromSideThreads) {
+ this.useFromSideThreads = useFromSideThreads;
+ return this;
+ }
+
+ /** Returns the current value set via {@link #setUseFromSideThreads}. */
+ public boolean getUseFromSideThreads() {
+ return useFromSideThreads;
+ }
+
+ /**
+ * How often {@link AuxIndexManager#onCreateWeight} actually samples searcher state for the
+ * dead-pair reaper; calls arriving sooner than this after the last accepted sample are skipped,
+ * since sampling is only a heuristic hint feeding the reap decision, not a correctness
+ * requirement. Default is one minute. Pass zero (or a non-positive value) to sample on every
+ * call.
+ */
+ public AuxIndexJoinConfig setSweepSamplingInterval(long duration, TimeUnit unit) {
+ this.sweepSamplingIntervalNanos = unit.toNanos(duration);
+ return this;
+ }
+
+ /** Returns the current value set via {@link #setSweepSamplingInterval}, in nanoseconds. */
+ public long getSweepSamplingIntervalNanos() {
+ return sweepSamplingIntervalNanos;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinMergePolicy.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinMergePolicy.java
new file mode 100644
index 000000000000..784b6b0d5c40
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinMergePolicy.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.FilterCodecReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergeTrigger;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.Bits;
+
+final class AuxIndexJoinMergePolicy extends MergePolicy {
+ @Override
+ public MergePolicy.MergeSpecification findMerges(
+ MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext)
+ throws IOException {
+ Set merging = mergeContext.getMergingSegments();
+ MergeSpecification spec = null;
+ for (SegmentCommitInfo info : segmentInfos) {
+ if (merging.contains(info)) {
+ continue;
+ }
+ Set pairFieldNames =
+ JoinIndexUtils.pairFieldNames(JoinIndexUtils.readFieldInfos(info));
+ if (!pairFieldNames.isEmpty()
+ && pendingPairRemovals.containsAll(
+ pairFieldNames)) { // todo sweep pending removals as well
+ if (spec == null) {
+ spec = new MergeSpecification();
+ }
+ spec.add(new DropSegmentMerge(List.of(info)));
+ }
+ }
+ return spec;
+ }
+
+ // counts merges that actually dropped a fully-dead segment; test-only observability, see
+ // droppedSegmentCount()
+ private final AtomicInteger droppedSegmentCount = new AtomicInteger();
+
+ /**
+ * A merge over a single dead segment whose contents are reported as fully deleted, so {@link
+ * IndexWriter} drops it instead of rewriting it -- see {@link #wrapForMerge}. Non-static so it
+ * can report back to the outer policy's {@link #droppedSegmentCount}.
+ */
+ private final class DropSegmentMerge extends OneMerge {
+ DropSegmentMerge(List segments) {
+ super(segments);
+ }
+
+ @Override
+ public CodecReader wrapForMerge(CodecReader reader) {
+ return new FilterCodecReader(reader) {
+ @Override
+ public CacheHelper getCoreCacheHelper() {
+ return reader.getCoreCacheHelper();
+ }
+
+ @Override
+ public CacheHelper getReaderCacheHelper() {
+ return null; // we are altering live docs
+ }
+
+ @Override
+ public Bits getLiveDocs() {
+ return new Bits.MatchNoBits(reader.maxDoc());
+ }
+
+ @Override
+ public int numDocs() {
+ return 0;
+ }
+ };
+ }
+
+ @Override
+ public void mergeFinished(boolean success, boolean segmentDropped) throws IOException {
+ if (segmentDropped) {
+ droppedSegmentCount.incrementAndGet();
+ }
+ super.mergeFinished(success, segmentDropped);
+ }
+ }
+
+ /** Test-only: how many sidecar segments this policy has actually reaped so far. */
+ int droppedSegmentCount() {
+ return droppedSegmentCount.get();
+ }
+
+ /** Test-only: how many dead pair field names are currently queued for the next reap. */
+ int pendingPairRemovalsCount() {
+ return pendingPairRemovals.size();
+ }
+
+ @Override
+ public MergePolicy.MergeSpecification findForcedMerges(
+ SegmentInfos segmentInfos,
+ int maxSegmentCount,
+ Map segmentsToMerge,
+ MergeContext mergeContext)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public MergePolicy.MergeSpecification findForcedDeletesMerges(
+ SegmentInfos segmentInfos, MergeContext mergeContext) throws IOException {
+ return null;
+ }
+
+ // caps how many distinct (from-searcher, to-searcher) pairs we remember snapshots for; a
+ // best-effort bound since this only anchors a heuristic reap, never correctness
+ private static final int MAX_TRACKED_SEARCHER_PAIRS = 256;
+ private final ConcurrentHashMap, Set>
+ lastNeededPairsBySearcherPair = new ConcurrentHashMap<>();
+ private final ConcurrentLinkedQueue> trackedSearcherPairsOrder =
+ new ConcurrentLinkedQueue<>();
+
+ // pair field names seen in an earlier snapshot but missing from a later one for the same
+ // (from-searcher, to-searcher) pair -- i.e. no longer needed -- queued here for findMerges to
+ // reap; also size-capped, same reasoning
+ private static final int MAX_PENDING_PAIR_REMOVALS = 4096;
+ private final Set pendingPairRemovals = ConcurrentHashMap.newKeySet();
+ private final ConcurrentLinkedQueue pendingPairRemovalsOrder =
+ new ConcurrentLinkedQueue<>();
+
+ // how often onCreateWeight actually bothers to sample searcher state; calls arriving sooner
+ // than this after the last accepted sample are skipped outright, since sampling is only a
+ // heuristic hint feeding findMerges' reap decision, not a correctness requirement. Zero (or
+ // negative) disables throttling entirely. Defaults to one minute; see #setSweepInterval.
+ private volatile long samplingIntervalNanos = TimeUnit.MINUTES.toNanos(1);
+
+ // Long.MIN_VALUE marks "never sampled yet" so the very first call always goes through,
+ // regardless of what System.nanoTime()'s arbitrary origin happens to be.
+ private final AtomicLong nextSampleAtNanos = new AtomicLong(Long.MIN_VALUE);
+
+ /**
+ * Configures how often {@link #onCreateWeight} actually samples searcher state for the dead-pair
+ * reaper; calls arriving sooner than this after the last accepted sample are skipped, since
+ * sampling is only a heuristic hint, not a correctness requirement. Defaults to one minute. Pass
+ * zero (or a non-positive value) to sample on every call.
+ */
+ void setSweepInterval(long duration, TimeUnit unit) {
+ this.samplingIntervalNanos = unit.toNanos(duration);
+ }
+
+ /**
+ * Approximates "has enough time passed since the last sample" with {@link System#nanoTime()} --
+ * the JDK's cheapest monotonic timer, since it need not track wall-clock time -- gated by a
+ * single CAS so that under concurrent callers exactly one wins a given interval and the rest
+ * skip, without any lock.
+ */
+ private boolean shouldSample() {
+ long interval = samplingIntervalNanos;
+ if (interval <= 0) {
+ return true;
+ }
+ long now = System.nanoTime();
+ long next = nextSampleAtNanos.get();
+ // subtraction (not direct comparison) so this stays correct across nanoTime() overflow, per
+ // its javadoc
+ if (next != Long.MIN_VALUE && now - next < 0) {
+ return false;
+ }
+ return nextSampleAtNanos.compareAndSet(next, now + interval);
+ }
+
+ void onCreateWeight(Set neededPairs, IndexSearcher fromSearcher, IndexSearcher searcher)
+ throws IOException {
+ if (!shouldSample()) {
+ return;
+ }
+ Object fromKey =
+ JoinIndexUtils.directoryKey(JoinIndexUtils.directory(fromSearcher.getIndexReader()));
+ Object toKey = JoinIndexUtils.directoryKey(JoinIndexUtils.directory(searcher.getIndexReader()));
+ Map.Entry searcherKey = Map.entry(fromKey, toKey);
+
+ Set currentSnapshot = Set.copyOf(neededPairs);
+ Set previousSnapshot =
+ AuxIndexJoinMergePolicy.putBounded(
+ lastNeededPairsBySearcherPair,
+ trackedSearcherPairsOrder,
+ searcherKey,
+ currentSnapshot,
+ MAX_TRACKED_SEARCHER_PAIRS);
+ if (previousSnapshot != null) {
+ for (String pairFieldName : previousSnapshot) {
+ if (!currentSnapshot.contains(pairFieldName)) {
+ AuxIndexJoinMergePolicy.addBounded(
+ pendingPairRemovals,
+ pendingPairRemovalsOrder,
+ pairFieldName,
+ MAX_PENDING_PAIR_REMOVALS);
+ }
+ }
+ }
+ }
+
+ /**
+ * Puts {@code key} -> {@code value}, evicting the oldest key(s) once {@code map} exceeds {@code
+ * maxSize}. Approximate under races (an eviction can drop a key concurrently re-inserted, or the
+ * map can briefly exceed {@code maxSize}) -- acceptable since callers only use this as a soft cap
+ * on a best-effort cache.
+ */
+ static V putBounded(
+ ConcurrentHashMap map,
+ ConcurrentLinkedQueue insertionOrder,
+ K key,
+ V value,
+ int maxSize) {
+ V previous = map.put(key, value);
+ if (previous == null) {
+ insertionOrder.add(key);
+ while (map.size() > maxSize) {
+ K oldest = insertionOrder.poll();
+ if (oldest == null) {
+ break;
+ }
+ map.remove(oldest);
+ }
+ }
+ return previous;
+ }
+
+ /** Same eviction policy as {@link AuxIndexJoinMergePolicy#putBounded}, for a plain set. */
+ static void addBounded(
+ Set set, ConcurrentLinkedQueue insertionOrder, T value, int maxSize) {
+ if (set.add(value)) {
+ insertionOrder.add(value);
+ while (set.size() > maxSize) {
+ T oldest = insertionOrder.poll();
+ if (oldest == null) {
+ break;
+ }
+ set.remove(oldest);
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinQuery.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinQuery.java
new file mode 100644
index 000000000000..297553b8f1c2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinQuery.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.function.Predicate;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.internal.hppc.IntHashSet;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Weight;
+import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.JoinSegmentReference;
+import org.jspecify.annotations.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Joins the from-side index to the to-side index this query is executed against, resolving
+ * from-side docs matching {@code fromQuery} to to-side docs through the auxiliary join index
+ * managed by {@link AuxIndexManager}: there, each (from-segment, to-segment) pair owns a
+ * SORTED_NUMERIC column named by both sides' persistent keys, whose doc number is the from-side doc
+ * id and whose value is the matching to-side doc id. Pair columns missing from the join index are
+ * built on demand at weight creation, so no explicit build step exists; obtain instances via {@link
+ * AuxIndexManager#newJoinQuery}. Matches score a constant.
+ */
+class AuxIndexJoinQuery extends Query {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ final AuxIndexManager joinIndex;
+ final String fromField;
+ final Query fromQuery;
+ protected final IndexSearcher fromSearcher;
+ final String toField;
+ private final ExecutorService fromExecutorService;
+
+ AuxIndexJoinQuery(
+ AuxIndexManager joinIndex,
+ String fromField,
+ Query fromQuery,
+ IndexSearcher fromSearcher,
+ String toField,
+ ExecutorService fromExecutorService) {
+ this.joinIndex = Objects.requireNonNull(joinIndex, "joinIndex");
+ this.fromField = Objects.requireNonNull(fromField, "fromField");
+ this.fromQuery = Objects.requireNonNull(fromQuery, "fromQuery");
+ this.fromSearcher = Objects.requireNonNull(fromSearcher, "fromSearcher");
+ this.toField = Objects.requireNonNull(toField, "toField");
+ this.fromExecutorService = fromExecutorService;
+ }
+
+ @SuppressWarnings("ReferenceEquality")
+ @Override
+ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
+ // the from-side selection rewrites against the from-side searcher, not against the (to-side)
+ // searcher this query is executed with
+ Query rewrittenFrom = fromQuery.rewrite(fromSearcher);
+ if (rewrittenFrom != fromQuery) { // TODO check MatchNoDocs ?
+ return new AuxIndexJoinQuery(
+ joinIndex, fromField, rewrittenFrom, fromSearcher, toField, fromExecutorService);
+ }
+ return super.rewrite(indexSearcher);
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher toSideSearcher, ScoreMode scoreMode, float boost)
+ throws IOException {
+ @NonNull Map neededPairs =
+ getRequiredColumNames(toSideSearcher);
+
+ joinIndex.onCreateWeight(neededPairs.keySet(), fromSearcher, toSideSearcher); // ignoring fields
+ Predicate isNeeded = neededPairs::containsKey;
+
+ Map existingJoinSegments;
+ IndexSearcher joinSearcher = this.joinIndex.acquire();
+ try {
+ existingJoinSegments = JoinIndexUtils.extractExistingJoinColumns(joinSearcher, isNeeded);
+ } finally {
+ this.joinIndex.release(joinSearcher);
+ }
+ int pairsNeeded = neededPairs.size();
+ neededPairs.keySet().removeAll(existingJoinSegments.keySet());
+ IntHashSet fromOrdsToLoad = new IntHashSet(neededPairs.size());
+ neededPairs.values().stream()
+ .mapToInt(AuxIndexManager.SegmentsTuple::fromLeafOrd)
+ .forEach(fromOrdsToLoad::add);
+ if (JoinIndexUtils.diagnosticsEnabled(log)) {
+ // pairsMissing > 0 on a repeat query means those pairs were never persisted by a previous
+ // run (writeBatch never captured them), so their from-segments' FK columns get reloaded
+ // here; pairsClaimed counts missing pairs another thread is building right now (claims are
+ // dropped once persisted), i.e. reloads that are pure waste
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=weight pairsNeeded={} pairsExisting={} pairsMissing={} pairsClaimed={}"
+ + " fkOrdsToLoad={} missingPairs={}",
+ pairsNeeded,
+ existingJoinSegments.size(),
+ neededPairs.size(),
+ joinIndex.countClaimedBuilds(neededPairs.keySet()),
+ fromOrdsToLoad.size(),
+ neededPairs.keySet());
+ }
+ Map> fromFutures = loadFromSide(fromOrdsToLoad);
+ // TODO this might produce too many small tasks
+ return new JoinIndexWeight(
+ this,
+ joinSearcher,
+ existingJoinSegments,
+ toSideSearcher.getIndexReader(),
+ scoreMode,
+ boost,
+ fromFutures);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map> loadFromSide(IntHashSet fromLeafsToLoad)
+ throws IOException {
+ LinkedHashMap> futuresByLeafOrds =
+ CollectionUtil.newLinkedHashMap(this.fromSearcher.getLeafContexts().size());
+ final Weight fromWeight =
+ this.fromSearcher.createWeight(this.fromQuery, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
+
+ List fromLeafs = new ArrayList<>(this.fromSearcher.getLeafContexts());
+ // heaviest first, the smallest last
+ fromLeafs.sort(
+ Comparator.comparing(ctx -> fromLeafsToLoad.contains(ctx.ord))
+ .thenComparingInt(ctx -> ctx.reader().maxDoc())
+ .reversed());
+
+ for (LeafReaderContext ctx : fromLeafs) {
+ futuresByLeafOrds.putLast(
+ ctx.ord, // the heaviest is submitted first, and accessed first as well
+ this.fromExecutorService.submit(
+ () ->
+ FromLeafJoinContext.heavyLoadFromLeaf(
+ fromWeight, fromField, ctx, fromLeafsToLoad.contains(ctx.ord))));
+ }
+
+ return futuresByLeafOrds;
+ }
+
+ private @NonNull Map getRequiredColumNames(
+ IndexSearcher searcher) {
+ Map neededPairs = new HashMap<>();
+ for (LeafReaderContext toContext : searcher.getIndexReader().leaves()) {
+ String toKey = JoinIndexUtils.getSideKey(toContext, toField);
+ for (LeafReaderContext fromCtx : fromSearcher.getLeafContexts()) {
+ String fromKey = JoinIndexUtils.getSideKey(fromCtx, fromField);
+ neededPairs.put(
+ fromKey + "_" + toKey, new AuxIndexManager.SegmentsTuple(fromCtx.ord, toContext.ord));
+ }
+ }
+ return neededPairs;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "AuxIndexJoinQuery(" + fromField + " -> " + toField + ", from: " + fromQuery + ")";
+ }
+
+ @Override
+ public void visit(QueryVisitor visitor) {
+ visitor.visitLeaf(this);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return sameClassAs(other) && equalsTo((AuxIndexJoinQuery) other);
+ }
+
+ private boolean equalsTo(AuxIndexJoinQuery other) {
+ // the join index and the from searcher compare by identity: a reopened from reader sees
+ // different ordinal spaces, so queries over different searcher instances must not be
+ // considered equal
+ return joinIndex == other.joinIndex
+ && fromSearcher == other.fromSearcher
+ && fromField.equals(other.fromField)
+ && fromQuery.equals(other.fromQuery)
+ && toField.equals(other.toField);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ classHash(),
+ System.identityHashCode(joinIndex),
+ fromField,
+ fromQuery,
+ System.identityHashCode(fromSearcher),
+ toField);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java
new file mode 100644
index 000000000000..fdc89164e142
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+
+/**
+ * The auxiliary join index: a self-maintaining sidecar persisting per (from-segment, to-segment)
+ * doc id mappings, so query-time joining reduces to bitset translation. It owns the sidecar's
+ * {@link IndexWriter} and {@link SearcherManager}; pair columns are built lazily when an {@link
+ * AuxIndexJoinQuery} first needs them, so users only construct an instance once, create queries
+ * with {@link #newJoinQuery} and search them with a bare to-side {@link IndexSearcher}:
+ *
+ *
+ * AuxIndexManager joinIndex = new AuxIndexManager(joinDir); // once per process
+ * Query q = joinIndex.newJoinQuery(fromField, fromQuery, fromSearcher, toField);
+ * TopDocs hits = toSearcher.search(q, 10);
+ * ...
+ * joinIndex.close(); // app shutdown
+ *
+ *
+ * After either side reopens, the next query builds only the missing (from, to) segment pairs:
+ * pair columns are addressed by both sides' persistent segment keys, which survive reopens. Pair
+ * columns orphaned by merges are not reclaimed yet; see package's javadoc.
+ */
+public final class AuxIndexManager implements Closeable {
+
+ private final IndexWriter writer;
+ private final SearcherManager manager;
+ private final JoinColumnIndexer pairBuilder = new JoinColumnIndexer(this);
+
+ // package-private (not private): tests reach in directly to observe the reaper's state
+ final AuxIndexJoinMergePolicy mergePolicy;
+ private final MergeScheduler mergeScheduler;
+ private final JoinColumWriter writerDelegate;
+ private final boolean blockingRefresh;
+ private final boolean useFromSideThreads;
+
+ /** A pair's (from-segment, to-segment) leaf ordinals. */
+ record SegmentsTuple(int fromLeafOrd, int toLeafOrd) {}
+
+ /**
+ * A pair column's address in the join index: the pair field name and the sidecar segment (name
+ * plus current leaf ordinal) carrying it -- enough to locate and open the column's real
+ * docvalues, or to check whether a pair already exists before deciding what still needs to be
+ * built. A resolved cell's edges are tracked separately, as a plain {@code DocEdges}, since they
+ * don't change as this reference is refreshed.
+ */
+ record JoinSegmentReference(
+ String pairFieldName, String joinSegmentName, int joinSegmentLeafOrd) {}
+
+ /**
+ * Opens a persistent auxiliary join index over the given directory, creating it if empty, using
+ * the default {@link AuxIndexJoinConfig}. The caller retains ownership of the directory: {@link
+ * #close()} does not close it.
+ */
+ public AuxIndexManager(Directory directory) throws IOException {
+ this(directory, new AuxIndexJoinConfig());
+ }
+
+ /**
+ * Opens a persistent auxiliary join index over the given directory, creating it if empty, using
+ * the given {@link AuxIndexJoinConfig}. The caller retains ownership of the directory: {@link
+ * #close()} does not close it.
+ */
+ public AuxIndexManager(Directory directory, AuxIndexJoinConfig config) throws IOException {
+ this(directory, config, new ConcurrentMergeScheduler());
+ }
+
+ /**
+ * Opens a persistent auxiliary join index over the given directory, creating it if empty, using
+ * the given {@link AuxIndexJoinConfig} and {@link MergeScheduler} in place of the default {@link
+ * ConcurrentMergeScheduler}. The caller retains ownership of the directory: {@link #close()} does
+ * not close it.
+ */
+ public AuxIndexManager(
+ Directory directory, AuxIndexJoinConfig config, MergeScheduler mergeScheduler)
+ throws IOException {
+ this.mergeScheduler = mergeScheduler;
+ this.mergePolicy = new AuxIndexJoinMergePolicy();
+ this.mergePolicy.setSweepInterval(config.getSweepSamplingIntervalNanos(), TimeUnit.NANOSECONDS);
+ this.writer =
+ new IndexWriter(
+ directory,
+ new IndexWriterConfig().setMergePolicy(mergePolicy).setMergeScheduler(mergeScheduler));
+ this.manager = new SearcherManager(writer, null);
+ JoinColumWriter bulkWriter = new JoinColumnDocWriter(); // new AIJoinColumnWriter()
+ this.writerDelegate =
+ config.getSingleFieldPerSegment()
+ ? new SingleColumnBySegmentWriter(bulkWriter)
+ : bulkWriter;
+ this.blockingRefresh = config.getBlockingRefresh();
+ this.useFromSideThreads = config.getUseFromSideThreads();
+ }
+
+ /**
+ * Creates a query joining the docs matching {@code fromQuery} in {@code fromSearcher}'s index to
+ * the index the returned query is executed against, through {@code fromField} = {@code toField}
+ * term equality. Missing pair columns are built into this join index on first execution.
+ *
+ * @deprecated use another constructor passing executor service
+ */
+ @Deprecated
+ public Query newJoinQuery(
+ String fromField, Query fromQuery, IndexSearcher fromSearcher, String toField) {
+ return newJoinQuery(fromField, fromQuery, fromSearcher, toField, new DirectExecutorService());
+ }
+
+ public Query newJoinQuery(
+ String fromField,
+ Query fromQuery,
+ IndexSearcher fromSearcher,
+ String toField,
+ ExecutorService fromExecutor) {
+ if (fromExecutor == null || !useFromSideThreads) {
+ fromExecutor = new DirectExecutorService();
+ }
+ return new AuxIndexJoinQuery(this, fromField, fromQuery, fromSearcher, toField, fromExecutor);
+ }
+
+ /**
+ * How many of the given pair field names have a build currently in flight (claims are removed
+ * once persisted). Diagnostic only; see {@link JoinColumnIndexer#countClaimedBuilds}.
+ */
+ int countClaimedBuilds(Set pairFieldNames) {
+ return pairBuilder.countClaimedBuilds(pairFieldNames);
+ }
+
+ IndexSearcher acquire() throws IOException {
+ return manager.acquire();
+ }
+
+ void release(IndexSearcher searcher) throws IOException {
+ manager.release(searcher);
+ }
+
+ /**
+ * Builds and persists the given missing pair columns, keyed by pair field name to their
+ * (from-segment, to-segment) leaf ordinals. Delegates to {@link
+ * JoinColumnIndexer#buildAndPersistJoinColumns}, which documents the claim/await dedup and the
+ * fresh-searcher double-check in detail.
+ *
+ * @param observedAbsentSearcher the join-index searcher in which the caller established that
+ * {@code missingPairs} are absent; possibly stale by now. Pass {@code null} when absence
+ * wasn't verified against a live searcher -- that forces the re-check scan.
+ * @return in memory data for just written segemts
+ */
+ Map buildAndPersistJoinColumns(
+ Map missingPairs,
+ IndexReader fromReader,
+ IndexReader toReader,
+ String toField,
+ String traceCtxId,
+ IndexSearcher observedAbsentSearcher,
+ Map> fromColumnFutures)
+ throws IOException, ExecutionException, InterruptedException {
+ return pairBuilder.buildAndPersistJoinColumns(
+ missingPairs,
+ fromReader,
+ toReader,
+ toField,
+ traceCtxId,
+ observedAbsentSearcher,
+ fromColumnFutures);
+ }
+
+ /**
+ * Serializes sidecar writes: one batch per commit keeps every batch at doc 0 of its own segment,
+ * preserving pair-column doc number == from-side doc id. Builders' futures are completed before
+ * this runs, so waiters consume the in-memory models without paying for the commit and refresh
+ * here -- a completed future does not mean the reader already exposes the column.
+ *
+ * It should be plain simple synchronized. As alternatives
+ *
+ *
+ * - push synchronized deeper to iw.addDocs(),iw.commit()
+ *
- merge columns from concurrent threads and writes them as a single batch - too much
+ * The invariant is a column starts ad doc#==0, this iw.addDocs(),iw.commit() goes one by
+ * one. But such segment might contain many parallel columns, since they have distinguishing
+ * names.
+ */
+ synchronized void writeBatch(Map mappings) throws IOException {
+ this.writerDelegate.writeJoinColumns(writer, mappings);
+ if (this.blockingRefresh) {
+ manager.maybeRefreshBlocking();
+ } else {
+ manager.maybeRefresh(); // perhaps it should be carried out the enclosing synchronize
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(manager, writer);
+ }
+
+ public void onCreateWeight(
+ Set neededPairs, IndexSearcher fromSearcher, IndexSearcher searcher)
+ throws IOException {
+ this.mergePolicy.onCreateWeight(neededPairs, fromSearcher, searcher);
+ }
+
+ /**
+ * Test-only: blocks until any in-flight merges (e.g. a dead-pair reap) finish. Only supported
+ * when this index was opened with a {@link ConcurrentMergeScheduler} (the default).
+ */
+ void waitForMerges() {
+ if (mergeScheduler instanceof ConcurrentMergeScheduler cms) {
+ cms.sync();
+ } else {
+ throw new UnsupportedOperationException(
+ "waitForMerges() requires a ConcurrentMergeScheduler, got " + mergeScheduler.getClass());
+ }
+ }
+
+ private static class SingleColumnBySegmentWriter extends JoinColumWriter {
+ private final JoinColumWriter bulkWriter;
+
+ public SingleColumnBySegmentWriter(JoinColumWriter bulkWriter) {
+ this.bulkWriter = bulkWriter;
+ }
+
+ @Override
+ void writeJoinColumns(IndexWriter writer, Map mappings)
+ throws IOException {
+ for (Map.Entry entry : mappings.entrySet()) {
+ bulkWriter.writeJoinColumns(writer, Map.of(entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/DirectExecutorService.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/DirectExecutorService.java
new file mode 100644
index 000000000000..a2cfefdd46d8
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/DirectExecutorService.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.util.List;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An {@link java.util.concurrent.ExecutorService} that runs every submitted task immediately on the
+ * calling thread, preserving the ordering and stack of the submitter. Useful to make asynchronous
+ * join work deterministic (e.g. in tests) or when the caller wants no background concurrency.
+ */
+final class DirectExecutorService extends AbstractExecutorService {
+
+ private volatile boolean shutdown;
+
+ @Override
+ public void shutdown() {
+ shutdown = true;
+ }
+
+ @Override
+ public List shutdownNow() {
+ shutdown = true;
+ return List.of();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return shutdown;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return shutdown;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return shutdown;
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ command.run();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/ForeignKeyColumn.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/ForeignKeyColumn.java
new file mode 100644
index 000000000000..e14cac329b68
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/ForeignKeyColumn.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * From-side state of one (from-segment, to-segment) pair: the from doc values, its live-docs mask,
+ * and the hashed from-side term dictionary. The hash maps each from-side term to its from-ord via
+ * {@link #fromOrdByHashOrd}, so the to-side stage of {@link JoinIndexUtils#computeDocMapping} can
+ * resolve a to-side term to the from-ord it shares. {@link #toDocByFromDoc} is zero-filled to
+ * {@code -1} and indexed by each live from-doc's from-ord by ; the to-side stage rewrites it in
+ * place from from-ords to to-docs.
+ */
+final class ForeignKeyColumn {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final BytesRefHash fromTermsHash;
+ private final int[] fromOrdByHashOrd;
+ private final int[] toDocByFromDoc;
+ private final int fromValuesCount;
+
+ public ForeignKeyColumn(LeafReaderContext fromContext, String fromField) throws IOException {
+ long startNanos = System.nanoTime();
+ SortedSetDocValues fromDV = DocValues.getSortedSet(fromContext.reader(), fromField);
+ Bits fromLiveDocs = fromContext.reader().getLiveDocs();
+ // dead code, kept until M:N support settles: the reverse (to-side sized) ord map was filled
+ // but never read
+ // long[] fromOrdByToOrd = new long[Math.toIntExact(toDV.getValueCount())];
+ // Arrays.fill(fromOrdByToOrd, -1L);
+ TermsEnum fromTerms = fromDV.termsEnum();
+ int fkVals = Math.toIntExact(fromDV.getValueCount());
+ // BytesRefHash only rehashes once it is exactly half full (count == hashHalfSize), so a table
+ // seeded at size 1 (fkVals == 1) never rehashes and ends up full; a probe for an absent term
+ // then never finds an empty slot and loops forever. Seeding at least 2 lets that rehash kick
+ // in.
+ int capacity =
+ Math.max(
+ 2,
+ BitUtil.nextHighestPowerOfTwo(
+ fkVals * 2 + 2 // don't want to rehash
+ ));
+ // pool,
+ BytesRefHash fromTermsHash =
+ new BytesRefHash(
+ new ByteBlockPool(new ByteBlockPool.DirectAllocator()),
+ capacity,
+ new BytesRefHash.DirectBytesStartArray(capacity));
+ int[] fromOrdByHashOrd = new int[capacity]; // should we fill it with -1 ? - probably shouldn't
+ for (BytesRef term = fromTerms.next(); term != null; term = fromTerms.next()) {
+ fromOrdByHashOrd[fromTermsHash.add(term)] = (int) fromTerms.ord();
+ }
+ // should we shrink it then? idk
+ int[] toDocByFromDoc = new int[fromContext.reader().maxDoc()];
+ Arrays.fill(toDocByFromDoc, -1);
+ for (int fromDoc = fromDV.nextDoc();
+ fromDoc != DocIdSetIterator.NO_MORE_DOCS;
+ fromDoc = fromDV.nextDoc()) {
+ if (fromLiveDocs != null && !fromLiveDocs.get(fromDoc)) {
+ continue;
+ }
+ toDocByFromDoc[fromDoc] = (int) fromDV.nextOrd();
+ }
+
+ this.fromTermsHash = fromTermsHash;
+ this.fromOrdByHashOrd = fromOrdByHashOrd;
+ this.toDocByFromDoc = toDocByFromDoc;
+ this.fromValuesCount = Math.toIntExact(fromDV.getValueCount());
+ if (JoinIndexUtils.diagnosticsEnabled(log)) {
+ // this constructor is the heavy from-side work (hashes the whole term dictionary), so every
+ // line here is one profiler-visible FK load; a segment recurring across queries means its
+ // pairs never get persisted and the load is being repeated in vain
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=fkload fromSeg={} field={} ord={} maxDoc={} values={} tookUs={}",
+ JoinIndexUtils.segmentName(fromContext),
+ fromField,
+ fromContext.ord,
+ fromContext.reader().maxDoc(),
+ fromValuesCount,
+ (System.nanoTime() - startNanos) / 1_000L);
+ }
+ }
+
+ public int getFromTermOrdOrDashOne(BytesRef value) {
+ int hashOrd = this.fromTermsHash.find(value);
+ if (hashOrd != -1) {
+ return this.fromOrdByHashOrd[hashOrd];
+ } else {
+ return -1;
+ }
+ }
+
+ /**
+ * per thread to-segment tasks uses these copies as a scratch, to turn values to "toDoc#" TODO
+ * presumably, we know how many copies we need, and the last call might return the original array
+ * which will be overridden by transformation.
+ */
+ public int[] cloneFromOrdByFromDoc() {
+ return toDocByFromDoc.clone();
+ }
+
+ public int fromSideMaxDocs() {
+ return toDocByFromDoc.length;
+ }
+
+ public int getFromValuesCount() {
+ return fromValuesCount;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/FromLeafJoinContext.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/FromLeafJoinContext.java
new file mode 100644
index 000000000000..ee396357e0b7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/FromLeafJoinContext.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.invoke.MethodHandles;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Weight;
+import org.jspecify.annotations.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class FromLeafJoinContext {
+ final JoinIndexUtils.CacheAndCount matches;
+ final ForeignKeyColumn fkColumn;
+ final int ord;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public FromLeafJoinContext(
+ JoinIndexUtils.CacheAndCount matches, ForeignKeyColumn fkColumn, int ord) {
+ this.matches = matches;
+ this.fkColumn = fkColumn;
+ this.ord = ord;
+ }
+
+ static @NonNull FromLeafJoinContext heavyLoadFromLeaf(
+ Weight fromWeight, String fromField, LeafReaderContext ctx, boolean loadFkColumn) {
+ try {
+ JoinIndexUtils.CacheAndCount docset = JoinIndexUtils.computeDocIdSet(fromWeight, ctx);
+ boolean loadFk = docset != null && docset.count() > 0 && loadFkColumn;
+ if (JoinIndexUtils.diagnosticsEnabled(log)) {
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=fromLeaf fromSeg={} ord={} fromMatches={} missingPair={}"
+ + " fkLoaded={}",
+ JoinIndexUtils.segmentName(ctx),
+ ctx.ord,
+ docset == null ? -1 : docset.count(),
+ loadFkColumn,
+ loadFk);
+ }
+ if (loadFk) {
+ // waste case: noone to-seg read FK,
+ return new FromLeafJoinContext(docset, new ForeignKeyColumn(ctx, fromField), ctx.ord);
+ } else {
+ return new FromLeafJoinContext(docset, null, ctx.ord);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumWriter.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumWriter.java
new file mode 100644
index 000000000000..d8430fa1318e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumWriter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+
+/**
+ * Isolates every write-side representation of a join batch behind this one write-only surface, so
+ * {@link AuxIndexManager} and {@link JoinIndexUtils} deal only in {@link JoinColumnModel} and never
+ * need to import a particular field-writing API themselves.
+ */
+abstract class JoinColumWriter {
+
+ protected JoinColumWriter() {}
+
+ /**
+ * Builds the pair columns for every entry in {@code mappings} (doc-map plus edges, keyed by pair
+ * field name) and writes them as a single indexing call + commit, so a batch's columns all land
+ * at doc 0 of their own sidecar segment.
+ */
+ abstract void writeJoinColumns(IndexWriter writer, Map mappings)
+ throws IOException;
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java
new file mode 100644
index 000000000000..3fcf8fe0e4dd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+
+/**
+ * Sibling of {@code AIJoinColumnWriter} writing the same pair columns through the plain {@link
+ * Document} / {@link IndexWriter#addDocuments} API instead of {@code
+ * org.apache.lucene.document.column}. The whole batch is built as one in-memory {@link List} of
+ * {@code batchNumDocs} documents and handed to a single {@link IndexWriter#addDocuments} call: per
+ * its block semantics that list is indexed atomically, with no flush allowed to land in the middle
+ * of it, so -- exactly like {@code AIJoinColumnWriter}'s single {@code addBatch} -- the whole batch
+ * is guaranteed to end up doc-for-doc (list index == doc id) in one sidecar segment, keeping doc
+ * 0's edges and every from-doc id aligned the same way.
+ */
+final class JoinColumnDocWriter extends JoinColumWriter {
+
+ JoinColumnDocWriter() {}
+
+ @Override
+ void writeJoinColumns(IndexWriter writer, Map mappings)
+ throws IOException {
+ int batchNumDocs = 0;
+ for (Map.Entry entry : mappings.entrySet()) {
+ batchNumDocs = Math.max(batchNumDocs, entry.getValue().maxDoc());
+ }
+ List docs = new ArrayList<>(batchNumDocs);
+ for (int i = 0; i < batchNumDocs; i++) {
+ docs.add(new Document());
+ }
+ for (Map.Entry entry : mappings.entrySet()) {
+ addJoinColumns(docs, entry.getValue(), entry.getKey());
+ }
+ // a single block: IndexWriter guarantees no intermediate flush splits it across segments
+ writer.addDocuments(docs);
+ writer.commit();
+ }
+
+ /**
+ * Adds one pair's fields to {@code docs}: the doc-map field resolving from-side doc ids to
+ * to-side doc ids, spread across the batch's docs, and the edges companion fields, always added
+ * to doc 0 even when the pair maps nothing, so a once-built pair is detectable in the join index
+ * and never rebuilt.
+ */
+ private static void addJoinColumns( // TODO don't write minusones columns for tombstones!!
+ List docs, JoinColumnModel mapping, String pairFieldName) throws IOException {
+ addOrdMap(docs, JoinIndexUtils.TO_DOC_VAL_BY_FROM_DOCNUM + pairFieldName, mapping);
+ addEdges(
+ docs, JoinIndexUtils.FROM_EDGES_PREFIX + pairFieldName, mapping.edges().fromDocEdges());
+ addEdges(docs, JoinIndexUtils.TO_EDGES_PREFIX + pairFieldName, mapping.edges().toDocEdges());
+ addEdges(
+ docs,
+ JoinIndexUtils.TO_COUNT_PREFIX + pairFieldName,
+ new int[] {mapping.edges().toCount()});
+ }
+
+ /**
+ * Adds a pair's {min, max} (or count) values to doc 0, mirroring {@code AIJoinColumnWriter}'s
+ * {@code edgesColumn}, which puts both values at doc 0 too.
+ */
+ private static void addEdges(List docs, String fieldName, int[] values) {
+ Document doc0 = docs.get(0);
+ for (int value : values) {
+ doc0.add(new SortedNumericDocValuesField(fieldName, value));
+ }
+ }
+
+ /**
+ * Adds the doc-map field: batch-local doc number is the from-side doc id and the SORTED_NUMERIC
+ * docvalue is the matching to-side doc id. From docs without a match get no value, hence the
+ * field is sparse.
+ */
+ private static void addOrdMap(List docs, String fieldName, JoinColumnModel mapping)
+ throws IOException {
+ SortedNumericDocValues values = mapping.toDocByFromDoc();
+ for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+ docs.get(doc).add(new SortedNumericDocValuesField(fieldName, values.nextValue()));
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnIndexer.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnIndexer.java
new file mode 100644
index 000000000000..2743d07ba86c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnIndexer.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.JoinSegmentReference;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.SegmentsTuple;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Builds missing pair columns for an {@link AuxIndexManager}: dedups concurrent builders per pair
+ * field name, computes the in-memory {@link JoinColumnModel}s, double-checks a fresh join-index
+ * searcher for pairs persisted since the caller looked, and persists what is still missing through
+ * {@link AuxIndexManager#writeBatch}.
+ */
+final class JoinColumnIndexer {
+
+ private final AuxIndexManager joinIndex;
+
+ /**
+ * Dedups concurrent builders per pair field name: the thread that installs the future writes the
+ * pair, others wait on it. A future completes as soon as the pair's in-memory {@link
+ * JoinColumnModel} is ready -- before the owning thread persists and refreshes -- so completion
+ * does not imply the column is visible through the index's searcher manager yet. Entries are
+ * transient: the owner removes its claims once the pair is persisted and refreshed (and on build
+ * failure), so the map only ever holds in-flight builds and never pins models in heap. A builder
+ * that races a not-yet-visible refresh and re-claims an already-persisted pair is stopped from
+ * writing a duplicate column by {@code #writeJoinSegments}'s fresh-searcher double-check, not by
+ * this map.
+ */
+ private final ConcurrentHashMap>>
+ pairBuilds = new ConcurrentHashMap<>();
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ JoinColumnIndexer(AuxIndexManager joinIndex) {
+ this.joinIndex = joinIndex;
+ }
+
+ /**
+ * How many of the given pair field names have a build currently in flight in {@link #pairBuilds}
+ * (claims are removed once persisted). Diagnostic only: a pair counted here but still reported
+ * missing by {@link JoinIndexUtils#extractExistingJoinColumns} means the caller is about to redo
+ * from-side work for a pair some other thread is building right now.
+ */
+ int countClaimedBuilds(Set pairFieldNames) {
+ int claimed = 0;
+ for (String pairFieldName : pairFieldNames) {
+ if (pairBuilds.containsKey(pairFieldName)) {
+ claimed++;
+ }
+ }
+ return claimed;
+ }
+
+ /**
+ * Builds and persists the given missing pair columns, keyed by pair field name to their
+ * (from-segment, to-segment) leaf ordinals. Pairs concurrently built by another thread are
+ * awaited, not rebuilt; awaiting yields the builder's in-memory model as soon as it is computed,
+ * possibly before the builder has committed it, so on return pairs built by this thread are
+ * refreshed into the internal searcher manager, while awaited ones may not be visible there yet.
+ *
+ *
Claims in {@link #pairBuilds} are transient -- the owner removes them once its pairs are
+ * persisted and refreshed, keeping the map bounded by in-flight builds. That makes re-claiming an
+ * already-persisted pair an expected event: the caller decided "missing" against {@code
+ * observedAbsentSearcher}, which may predate another thread's completed build. So after claiming,
+ * this method re-checks a fresh searcher and skips persisting any pair that turns out already
+ * written. Since only the claim owner ever writes a pair, absence in a post-claim fresh searcher
+ * is conclusive, so no duplicate pair column can be written; rediscovered pairs still cost their
+ * in-memory recompute, which the caller gets back in the result like any built pair. As a
+ * shortcut, when {@link AuxIndexManager#acquire()} returns {@code observedAbsentSearcher} itself,
+ * nothing was committed since that searcher was current, and the re-check scan is skipped.
+ *
+ * @param observedAbsentSearcher the join-index searcher in which the caller established that
+ * {@code missingPairs} are absent; possibly stale by now. Pass {@code null} when absence
+ * wasn't verified against a live searcher -- that forces the re-check scan.
+ * @return in memory data for just written segemts
+ */
+ Map buildAndPersistJoinColumns(
+ Map missingPairs,
+ IndexReader fromReader,
+ IndexReader toReader,
+ String toField,
+ String traceCtxId,
+ IndexSearcher observedAbsentSearcher,
+ Map> fromColumnFutures)
+ throws IOException, ExecutionException, InterruptedException {
+ long startNanos = System.nanoTime();
+ Claims claims = claimPairs(missingPairs.keySet());
+ Map loadedMappings = Map.of();
+ Set writtenPairs = Set.of();
+ try {
+ if (!claims.owned().isEmpty()) {
+ loadedMappings =
+ computeOwnedModels(
+ claims.owned().keySet(),
+ missingPairs,
+ fromReader,
+ toReader,
+ toField,
+ fromColumnFutures);
+ completeClaimFutures(claims.owned(), loadedMappings);
+ Map unwrittenMappings =
+ dropAlreadyPersisted(loadedMappings, claims.owned().keySet(), observedAbsentSearcher);
+ persistBatch(unwrittenMappings);
+ writtenPairs = unwrittenMappings.keySet();
+ releaseClaims(claims.owned());
+ }
+ } catch (Throwable t) {
+ withdrawClaims(claims.owned(), t);
+ throw t;
+ }
+ long builtNanos = System.nanoTime() - startNanos;
+ Map result = new LinkedHashMap<>(loadedMappings);
+ collectAwaited(claims.awaited(), result);
+ logBuildDiagnostics(
+ traceCtxId,
+ missingPairs.size(),
+ loadedMappings,
+ claims.awaited().size(),
+ writtenPairs,
+ startNanos,
+ builtNanos);
+ return result;
+ }
+
+ /**
+ * The claim/await split of one build request: {@code owned} holds the futures this thread
+ * installed into {@link #pairBuilds} (it must build, persist, and eventually remove them), {@code
+ * awaited} the futures of pairs some other thread is already building.
+ */
+ private record Claims(
+ Map>> owned,
+ List>> awaited) {}
+
+ /**
+ * Races a claim into {@link #pairBuilds} for every requested pair: the thread whose future lands
+ * owns that pair's build, a pair already claimed by another thread is awaited instead.
+ */
+ private Claims claimPairs(Set pairFieldNames) {
+ Map>> owned =
+ new LinkedHashMap<>();
+ List>> awaited = new ArrayList<>();
+ for (String pairFieldName : pairFieldNames) {
+ CompletableFuture> created = new CompletableFuture<>();
+ CompletableFuture> existing =
+ pairBuilds.putIfAbsent(pairFieldName, created);
+ if (existing == null) {
+ owned.put(pairFieldName, created);
+ } else {
+ awaited.add(existing);
+ }
+ }
+ return new Claims(owned, awaited);
+ }
+
+ /** Computes the in-memory {@link JoinColumnModel} for every owned pair. */
+ private static Map computeOwnedModels(
+ Set ownedPairs,
+ Map missingPairs,
+ IndexReader fromReader,
+ IndexReader toReader,
+ String toField,
+ Map> fromColumnFutures)
+ throws IOException, ExecutionException, InterruptedException {
+ Map loadedMappings = new LinkedHashMap<>();
+ JoinIndexUtils.ToDocInvertor toInvertor = new JoinIndexUtils.ToDocInvertor(toField);
+ for (String pairFieldName : ownedPairs) {
+ SegmentsTuple position = missingPairs.get(pairFieldName);
+ LeafReaderContext toContext = toReader.leaves().get(position.toLeafOrd());
+ LeafReaderContext fromContext = fromReader.leaves().get(position.fromLeafOrd());
+ Future fromLeafJoinContextFuture =
+ fromColumnFutures.get(fromContext.ord);
+ assert fromLeafJoinContextFuture != null;
+ FromLeafJoinContext fkCtx = fromLeafJoinContextFuture.get();
+ JoinIndexUtils.JoinColumnModel mapping =
+ JoinIndexUtils.computeDocMapping(toContext, toField, fkCtx.fkColumn, toInvertor);
+ loadedMappings.put(pairFieldName, mapping);
+ }
+ return loadedMappings;
+ }
+
+ /**
+ * Completes every owned claim future with its computed model. Runs before {@link #persistBatch}:
+ * waiters only need the in-memory model, so they don't pay for the owner's commit + refresh;
+ * completion does NOT imply the column is visible through the searcher manager yet.
+ */
+ private static void completeClaimFutures(
+ Map>> owned,
+ Map loadedMappings) {
+ for (Map.Entry>> entry :
+ owned.entrySet()) {
+ entry
+ .getValue()
+ .complete(
+ new AbstractMap.SimpleImmutableEntry<>(
+ entry.getKey(), loadedMappings.get(entry.getKey())));
+ }
+ }
+
+ /**
+ * Double-checks the owned pairs against a fresh searcher, returning only the mappings still
+ * unwritten: claims are dropped once persisted, so a caller deciding "missing" on a stale
+ * searcher can re-claim an already-persisted pair; owning the claim means no one else can write
+ * it concurrently, so what the fresh searcher lacks is conclusively unwritten.
+ *
+ * It's a race condition tradeoff:
+ *
+ *
+ * - this thread didn't see a column and decided to compete for it.
+ *
- another thread might have persisted it in the meantime, and removed it's claim from
+ * pairBuilds.
+ *
- thread 1. calculate a join column model, and ready to write it
+ *
- if a column occur in already persisted: it skip wtiting it, but returns the calculated
+ * memory model to the caller
+ *
+ *
+ * overall, under high concurrency we waste some computation for sweeping pairBuilds;
+ * alternatively we need to invent a way to remove entries from it somewhere later without a race
+ */
+ private Map dropAlreadyPersisted(
+ Map loadedMappings,
+ Set ownedPairs,
+ IndexSearcher observedAbsentSearcher)
+ throws IOException {
+ Map unwrittenMappings = loadedMappings;
+ IndexSearcher freshJoinSearcher = joinIndex.acquire();
+ try {
+ if (freshJoinSearcher != observedAbsentSearcher) {
+ Map alreadyPersisted =
+ JoinIndexUtils.extractExistingJoinColumns(freshJoinSearcher, ownedPairs::contains);
+ if (!alreadyPersisted.isEmpty()) {
+ unwrittenMappings = new LinkedHashMap<>(loadedMappings);
+ unwrittenMappings.keySet().removeAll(alreadyPersisted.keySet());
+ }
+ } // else: same searcher instance the caller saw the pairs absent in -- nothing was
+ // committed since, so the pairs are provably unwritten and the scan is redundant
+ } finally {
+ joinIndex.release(freshJoinSearcher);
+ }
+ return unwrittenMappings;
+ }
+
+ /**
+ * Persists the still-unwritten mappings, if any. All pairs of a batch go in together: pair
+ * columns are addressed by from-side doc id, so a batch must start at doc 0 of its sidecar
+ * segment, which {@link AuxIndexManager#writeBatch} guarantees by flushing one batch per commit.
+ */
+ // TODO it can be more asynchronous, besides of pairBuilds removal - and it's a problem
+ // would be great if many concurrent threads merge lists and write it at once,
+ // since it's synchronized bottleneck for now.
+ // and the thread don't even need to wait until writeBatch() is done (beside of pairBuilds
+ // removal, you know)
+ // TODO flush every single field to get single field segments
+ private void persistBatch(Map unwrittenMappings) throws IOException {
+ if (!unwrittenMappings.isEmpty()) {
+ joinIndex.writeBatch(unwrittenMappings);
+ }
+ }
+
+ /**
+ * Drops the owned claims after a successful build: every owned pair is now persisted (just
+ * written, or found persisted by the double-check) and visible to any searcher acquired from here
+ * on, and duplicate writes are prevented by the double-check, not by retained entries -- dropping
+ * them keeps {@link #pairBuilds} bounded by in-flight builds and unpins the models from heap.
+ * Waiters already holding the futures are unaffected.
+ */
+ private void releaseClaims(
+ Map>> owned) {
+ for (Map.Entry>> entry :
+ owned.entrySet()) {
+ pairBuilds.remove(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Withdraws the owned claims on failure so a later query can retry the build; {@code
+ * completeExceptionally} only takes effect when the failure happened before {@link
+ * #completeClaimFutures} -- if writeBatch failed, waiters have already observed success with a
+ * valid in-memory model, and only the withdrawn claim (and the caller's rethrow) records that the
+ * column was never persisted.
+ */
+ private void withdrawClaims(
+ Map>> owned, Throwable t) {
+ for (Map.Entry>> entry :
+ owned.entrySet()) {
+ pairBuilds.remove(entry.getKey(), entry.getValue());
+ entry.getValue().completeExceptionally(t);
+ }
+ }
+
+ /**
+ * Joins the pairs claimed by other threads and merges their models into {@code result},
+ * unwrapping a builder's failure into this thread's checked contract.
+ */
+ private static void collectAwaited(
+ List>> awaited,
+ Map result)
+ throws IOException {
+ for (CompletableFuture> future : awaited) {
+ try {
+ Map.Entry entry = future.join();
+ result.put(entry.getKey(), entry.getValue());
+ } catch (CompletionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException ioe) {
+ throw ioe;
+ }
+ if (cause instanceof RuntimeException re) {
+ throw re;
+ }
+ throw new IOException(cause);
+ }
+ }
+ }
+
+ /**
+ * Emits the {@code evt=build} diagnostic line. The built/awaited split matters: an awaited pair
+ * cost this thread only the wait, so folding the two together would attribute another thread's
+ * build work to this query; pairsBuilt counts models computed by this thread, writtenPairs what
+ * actually reached the sidecar -- the difference is pairs the double-check found already
+ * persisted.
+ */
+ private static void logBuildDiagnostics(
+ String traceCtxId,
+ int pairsRequested,
+ Map loadedMappings,
+ int pairsAwaited,
+ Set writtenPairs,
+ long startNanos,
+ long builtNanos) {
+ if (!JoinIndexUtils.diagnosticsEnabled(log) || pairsRequested == 0) {
+ return;
+ }
+ long toCount = 0;
+ for (JoinColumnModel model : loadedMappings.values()) {
+ toCount += model.edges().toCount();
+ }
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=build ctx={} pairsRequested={} pairsBuilt={} pairsAwaited={}"
+ + " builtMs={} awaitedMs={} toCount={} writtenPairs={}",
+ traceCtxId == null ? "-" : traceCtxId,
+ pairsRequested,
+ loadedMappings.size(),
+ pairsAwaited,
+ builtNanos / 1_000_000L,
+ (System.nanoTime() - startNanos - builtNanos) / 1_000_000L,
+ toCount,
+ writtenPairs);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java
new file mode 100644
index 000000000000..091f7439fb79
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java
@@ -0,0 +1,784 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.JoinSegmentReference;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.SegmentsTuple;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.DocEdges;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Joins all from-side segments to the single to-side segment.
+ *
+ * Instrumentation
+ *
+ * Emits TRACE lines prefixed {@code AUXIJOIN} in logfmt (space-separated {@code key=value}), one
+ * {@code evt=} per line, so a run can be parsed without a logging-config change:
+ *
+ *
+ * - {@code evt=build} -- a join-index write happened; carries the wall time it cost. Emitted by
+ * {@code AuxIndexManager#writeJoinSegments}, the chokepoint both build paths share: {@code
+ * cause=eager-create-weight} for the bulk build {nolink AuxIndexManager#ensureJoinSegments}
+ * does at {@link AuxIndexJoinQuery#createWeight} time, and {@code cause=lazy-to-segment} for
+ * the per-context fallback below. In a steady run the eager path does all the work and the
+ * lazy one never fires, so a log with no {@code cause=lazy-to-segment} line is the expected
+ * shape.
+ *
- {@code evt=ctx} -- this context finished setting up: how many (from, to) pairs contributed,
+ * how many the a-priori from-edge check dropped before any column was opened, and how loose
+ * the resulting approximation is.
+ *
- {@code evt=drain} -- one join column was read through during confirmation, and whether that
+ * read confirmed the doc under test (an early exit) or not.
+ *
- {@code evt=done} -- confirmation reached a terminal state for this context: every column
+ * has been drained, so the half-read union has converged. Carries the per-context totals.
+ *
+ *
+ * Every line carries {@code ctx=}, the {@link #ctxId} of the context it belongs to ({@code -} on an
+ * eager build, which precedes every context), so a parser can attribute drains exactly instead of
+ * assuming one context per to-segment is alive at a time.
+ *
+ * The counters are best-effort: they are plain fields on a context confined to one to-segment,
+ * and so to one search thread, but nothing enforces that. A context whose confirmation never
+ * converges emits no {@code evt=done} line -- absence of one is itself the signal that laziness
+ * paid off for that segment, at the cost of that context's final counters going unreported.
+ */
+class JoinIndexScorerSupplier extends ScorerSupplier {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Per-JVM sequence behind {@link #ctxId}. Only has to distinguish contexts alive at the same
+ * time, so wrapping after 2^63 is not a concern; base 36 just keeps the log lines short.
+ */
+ private static final AtomicLong CTX_SEQ = new AtomicLong();
+
+ /**
+ * Identifies this context in the log. Every {@code evt=ctx} / {@code evt=drain} / {@code
+ * evt=done} line carries it, so drains can be attributed to their context exactly, rather than by
+ * assuming only one context per to-segment is alive -- which only holds when a single query runs
+ * at a time.
+ */
+ private final String ctxId = Long.toString(CTX_SEQ.incrementAndGet(), 36);
+
+ final LeafReaderContext toContext;
+ final Query fromQuery;
+ final IndexSearcher fromSearcher;
+ private final Map> fromColumnFutures;
+ private final ScoreMode scoreMode;
+ private final float boost;
+ private IndexSearcher lastSeenJoinSearcher;
+ final String fromField;
+ final String toField;
+ private final AuxIndexManager joinIndex;
+
+ // TODO all of these might be final since they are set in the constructor
+ private int firstToDoc = DocIdSetIterator.NO_MORE_DOCS;
+ private int lastToDoc = -1;
+ private long matchedToDocsCount = 0;
+ private FixedBitSet falsePositiveToDocsBits = null;
+
+ // ---- instrumentation, best effort; see the class javadoc for the emitted format ----
+ /** nanos spent inside {@code AuxIndexManager#writeJoinSegments} on behalf of this context */
+ private long joinIndexBuildNanos;
+
+ /** pairs that had a from-side match before the a-priori from-edge check ran */
+ private int joinLeafsCreated;
+
+ /** pairs the a-priori from-edge check dropped, i.e. columns never opened at all */
+ private int leafsDroppedApriori;
+
+ /** calls to {@link LazyConfirmationIterator#matches()} */
+ private int confirmCalls;
+
+ /** those answered from the half-read union alone, with no column read */
+ private int confirmFreeHits;
+
+ /** columns read through during confirmation */
+ private int leafsDrained;
+
+ /** from-docs walked while draining, i.e. the column reads laziness is trying to avoid */
+ private long fromDocsWalked;
+
+ /** set once the {@code evt=done} line has been emitted, so it is emitted at most once */
+ private boolean reported;
+
+ /** ordered by from-cost descending */
+ private final List leafJoins = new ArrayList<>();
+
+ private final IndexReader toReader;
+
+ private final class LazyConfirmationIterator extends TwoPhaseIterator {
+ FixedBitSet falseNegToDocsBits = null;
+ private int shift;
+
+ private LazyConfirmationIterator(DocIdSetIterator approximation) {
+ super(approximation);
+ }
+
+ @Override
+ public boolean matches() throws IOException {
+ confirmCalls++;
+ if (leafJoins.isEmpty()) {
+ assert falsePositiveToDocsBits.get(approximation.docID());
+ confirmFreeHits++;
+ return true; /// aproximation is a refined true pos already
+ }
+ if (falseNegToDocsBits != null) {
+ if (falseNegToDocsBits.get(approximation.docID() - shift)) {
+ confirmFreeHits++;
+ return true;
+ } // otherwise we don't know if 0 is real false
+ }
+
+ IndexSearcher freshSearcher = JoinIndexScorerSupplier.this.joinIndex.acquire();
+ try {
+ refreshJoinTasksReferences(freshSearcher);
+ assert JoinIndexScorerSupplier.this.lastSeenJoinSearcher == freshSearcher;
+ for (LeafJoin joinTask : new ArrayList<>(leafJoins)) {
+ if (falseNegToDocsBits == null) {
+ this.shift = approximation.docID();
+ falseNegToDocsBits = new FixedBitSet(lastToDoc + 1 - shift);
+ }
+ int walked = joinTask.dumpMatchesInto(falseNegToDocsBits, shift, approximation.docID());
+ if (!joinTask.fromSegIterIsNotExausted()) {
+ JoinIndexScorerSupplier.this.dropJoinLeaf(joinTask);
+ leafsDrained++;
+ }
+ fromDocsWalked += walked;
+ if (!leafJoins.isEmpty()) {
+ if (falseNegToDocsBits.get(approximation.docID() - shift)) {
+ logDrain(joinTask, walked, true);
+ return true;
+ } // otherwise we don't know if 0 is real false
+ }
+ logDrain(joinTask, walked, false);
+ }
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } finally {
+ JoinIndexScorerSupplier.this.joinIndex.release(freshSearcher);
+ }
+ // drop all to masks, got no hit - it means it's a true negative now.
+
+ falsePositiveToDocsBits.clear(shift, lastToDoc + 1);
+ FixedBitSet.orRange(
+ falseNegToDocsBits, 0, falsePositiveToDocsBits, shift, lastToDoc - shift + 1);
+
+ boolean matched = falsePositiveToDocsBits.get(approximation.docID());
+ // every column has now been drained: the half-read union has converged, so from here on
+ // every answer -- true or false -- is a free lookup and there is nothing left to prune
+ logConfirmationDone(matched ? "converged-on-match" : "converged-on-refutation");
+ return matched;
+ }
+
+ /**
+ * Reports one column read. {@code confirmed=true} is the early exit the lazy variant exists
+ * for: the doc under test was found before the remaining {@code cellsLeft} columns were
+ * touched.
+ */
+ private void logDrain(LeafJoin joinTask, int walked, boolean confirmed) {
+ if (!JoinIndexUtils.diagnosticsEnabled(log)) {
+ return;
+ }
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=drain ctx={} toSeg={} pair={} confirmed={} walked={} colToCount={}"
+ + " cellsLeft={} hCard={} confirmCalls={}",
+ ctxId,
+ JoinIndexUtils.segmentName(toContext),
+ joinTask.pairFieldName,
+ confirmed,
+ walked,
+ joinTask.toCount(),
+ leafJoins.size(),
+ falseNegToDocsBits == null ? 0 : falseNegToDocsBits.cardinality(),
+ confirmCalls);
+ }
+
+ @Override
+ public float matchCost() {
+ return matchedToDocsCount;
+ }
+ }
+
+ /**
+ * Represents a cell in the join matrix bounded to from and to segments. Resolved exactly once,
+ * either from the join index ({@link #resolveFromReadingJoinIndex}: just the pair's {@link
+ * DocEdges}, with real docvalues opened later, on demand, through {@link #joinSegmentRef} --
+ * which keeps being refreshed as the join reader reopens -- or from the indexer ({@link
+ * #resolveFromCalculatedModel}: the pair's edges plus an in-memory {@link JoinColumnModel} that
+ * needs no further indirection to read.
+ */
+ class LeafJoin implements DocEdges {
+ final String pairFieldName;
+ final DocIdSetIterator fromSegmentDocIdIter;
+ final long fromMatchCount;
+ JoinSegmentReference joinSegmentRef;
+ final SegmentsTuple segmentsFromTo;
+ private DocEdges edges;
+ // set only when resolved from the indexer; null means real docvalues are opened through
+ // joinSegmentRef instead
+ private JoinColumnModel docMapping;
+
+ LeafJoin(
+ String pairFieldName,
+ SegmentsTuple segmentsFromTo,
+ DocIdSetIterator fromSegmentDocIdIter,
+ long fromMatchCount) {
+ this.pairFieldName = pairFieldName;
+ this.segmentsFromTo = segmentsFromTo;
+ this.fromSegmentDocIdIter = fromSegmentDocIdIter;
+ this.fromMatchCount = fromMatchCount;
+ assert fromSegmentDocIdIter.docID() != DocIdSetIterator.NO_MORE_DOCS;
+ assert fromSegmentDocIdIter.docID() >= 0;
+ }
+
+ /** Resolves this cell, once, to a join-index-persisted pair's edges. */
+ void resolveFromReadingJoinIndex(DocEdges edges) {
+ assert this.edges == null : "already resolved: " + this.edges;
+ this.edges = edges;
+ }
+
+ /** Resolves this cell, once, to an in-memory doc mapping built on demand. */
+ void resolveFromCalculatedModel(JoinColumnModel docMapping) {
+ assert this.edges == null : "already resolved: " + this.edges;
+ this.edges = docMapping.edges();
+ this.docMapping = docMapping;
+ }
+
+ boolean isResolved() {
+ return edges != null;
+ }
+
+ SortedNumericDocValues toDocsByFromDocsDV() throws IOException {
+ assert edges != null : "not resolved yet: " + this;
+ if (docMapping != null) {
+ // built on demand, so it's available directly, with no searcher involved
+ return docMapping.toDocByFromDoc();
+ }
+ // resolved from the join index: joinSegmentRef locates the real, on-disk column
+ LeafReaderContext joinContext =
+ lastSeenJoinSearcher.getLeafContexts().get(joinSegmentRef.joinSegmentLeafOrd());
+ assert JoinIndexUtils.segmentName(joinContext).equals(joinSegmentRef.joinSegmentName());
+ return joinContext
+ .reader()
+ .getSortedNumericDocValues(
+ JoinIndexUtils.TO_DOC_VAL_BY_FROM_DOCNUM + joinSegmentRef.pairFieldName());
+ }
+
+ /**
+ * Walks this cell's from-iterator from its current (prepositioned) doc through its edges' last
+ * from-doc, setting every to-doc it maps to -- shifted by {@code shift} -- in {@code
+ * matchedToDocs}.
+ *
+ * @return how many from-docs were walked, i.e. the column-read work this drain cost
+ */
+ int dumpMatchesInto(FixedBitSet matchedToDocs, int shift, int earlyExitDoc) throws IOException {
+ SortedNumericDocValues toDocsByFromDoc = toDocsByFromDocsDV();
+ int walked = 0;
+ boolean confirmedCurrent = false;
+ for (int fromDoc = fromSegmentDocIdIter.docID(); // prepositioned to the first match
+ fromSegIterIsNotExausted();
+ fromDoc = fromSegmentDocIdIter.nextDoc()) {
+ walked++;
+ if (toDocsByFromDoc.advanceExact(
+ fromDoc)) { // TODO will it be faster to use advance() and leapfrog?
+ for (int i = 0; i < toDocsByFromDoc.docValueCount(); i++) {
+ int toDocMatch = (int) toDocsByFromDoc.nextValue();
+ assert toDocMatch <= toDocEdges()[1] && toDocMatch >= toDocEdges()[0]
+ : "to doc "
+ + toDocMatch
+ + " above edges union max "
+ + Arrays.toString(toDocEdges());
+ // shift is wherever the approximation iterator first landed, which need not be
+ // the global firstToDoc (e.g. under a boolean conjunction); a match below shift
+ // is unreachable -- the iterator only moves forward -- so it's dropped rather
+ // than written at a negative offset
+ // Also. we don't need toDocs less than one we're confirming currently
+ if (toDocMatch >= shift && toDocMatch >= earlyExitDoc) {
+ matchedToDocs.set(toDocMatch - shift);
+ if (toDocMatch == earlyExitDoc) {
+ confirmedCurrent = true;
+ }
+ }
+ }
+ if (confirmedCurrent) {
+ // we've just confirmed the doc, which was matches() is called for
+ return walked; // giveup
+ }
+ }
+ }
+ return walked;
+ }
+
+ private boolean fromSegIterIsNotExausted() {
+ return fromSegmentDocIdIter.docID() != DocIdSetIterator.NO_MORE_DOCS
+ && fromSegmentDocIdIter.docID() <= fromDocEdges()[1];
+ }
+
+ @Override
+ public int[] fromDocEdges() {
+ return edges.fromDocEdges();
+ }
+
+ @Override
+ public int[] toDocEdges() {
+ return edges.toDocEdges();
+ }
+
+ @Override
+ public int toCount() {
+ return edges.toCount();
+ }
+ }
+
+ /** Appends {@code cell} to {@link #leafJoins} and registers it in both indices. */
+ private LeafJoin addJoinLeaf(LeafJoin cell) {
+ leafJoins.add(cell);
+ return cell;
+ }
+
+ /**
+ * Drops {@code cell} from {@link #leafJoins}. It's a little bit awkward. Every iteration join
+ * task list is copied and then removed from original list via reference equality. Ideally,
+ * iterators should be used for removals.
+ */
+ private void dropJoinLeaf(LeafJoin cell) {
+ leafJoins.remove(cell);
+ }
+
+ record TaskRefreshResult(
+ Set> joinSegments,
+ Set> justWritten) {}
+
+ /**
+ * @param weightAgeJoinSegmentsReadOnly the join segments cached at {@link
+ * AuxIndexJoinQuery#createWeight} time DON'T MODIFY ME!!!
+ * @param weightAgeJoinSearcher the join searcher cached at {@link AuxIndexJoinQuery#createWeight}
+ * time
+ * @param scorerSupplierAgeJoinSearcher the join searcher used at {@link
+ * JoinIndexWeight#scorerSupplier} time
+ * @param fromColumnFutures from side data by segment ord.
+ */
+ JoinIndexScorerSupplier(
+ LeafReaderContext toContext,
+ String fromField,
+ Query fromQuery,
+ IndexSearcher fromSearcher,
+ String toField,
+ IndexReader toReader,
+ Map weightAgeJoinSegmentsReadOnly,
+ IndexSearcher weightAgeJoinSearcher,
+ IndexSearcher scorerSupplierAgeJoinSearcher,
+ AuxIndexManager joinIndex,
+ Map> fromColumnFutures,
+ ScoreMode scoreMode,
+ float boost)
+ throws ExecutionException, InterruptedException, IOException {
+ this.toContext = toContext;
+ this.fromField = fromField;
+ this.fromQuery = fromQuery;
+ this.fromSearcher = fromSearcher;
+ this.toField = toField;
+ this.toReader = toReader;
+ this.fromColumnFutures = fromColumnFutures;
+ this.joinIndex = joinIndex;
+ this.scoreMode = scoreMode;
+ this.boost = boost;
+
+ // 1. check from scorers
+ List fromItersTasks = createLeafJoins();
+ for (LeafJoin newJoinTask : fromItersTasks) {
+ this.joinLeafsCreated++;
+ this.addJoinLeaf(newJoinTask);
+ // 2. set old segment references
+ JoinSegmentReference oldReference =
+ weightAgeJoinSegmentsReadOnly.get(newJoinTask.pairFieldName);
+ if (oldReference != null) {
+ newJoinTask.joinSegmentRef = oldReference;
+ }
+ }
+
+ this.lastSeenJoinSearcher =
+ weightAgeJoinSearcher; // set old searcher, it corresponds to weightAgeJoinSegmentsReadOnly
+ TaskRefreshResult refreshedAndNew = refreshJoinTasksReferences(scorerSupplierAgeJoinSearcher);
+ for (Entry entry : refreshedAndNew.joinSegments) {
+ LeafJoin task = entry.getKey();
+ LeafReaderContext joinLeaf = entry.getValue(); // got it from searcher leafs by refOrd
+ assert JoinIndexUtils.segmentName(joinLeaf).equals(task.joinSegmentRef.joinSegmentName());
+ assert joinLeaf.ord == task.joinSegmentRef.joinSegmentLeafOrd();
+ task.resolveFromReadingJoinIndex( // ok. this one may be ready for search.
+ new JoinIndexUtils.Edges(
+ JoinIndexUtils.loadEdges(
+ joinLeaf, JoinIndexUtils.FROM_EDGES_PREFIX + task.pairFieldName),
+ // TODO it might not need to be loaded, if "from" edges fully cut of the column. Thus,
+ // we won't read even "to" edges at all.
+ JoinIndexUtils.loadEdges(
+ joinLeaf, JoinIndexUtils.TO_EDGES_PREFIX + task.pairFieldName),
+ // TODO use it for ordering join segment iteration, desc
+ JoinIndexUtils.loadEdges(
+ joinLeaf, JoinIndexUtils.TO_COUNT_PREFIX + task.pairFieldName)[0]));
+ }
+ for (Entry entry : refreshedAndNew.justWritten) {
+ LeafJoin task = entry.getKey();
+ task.resolveFromCalculatedModel(entry.getValue());
+ }
+ // edges are loaded
+ for (LeafJoin task :
+ List.copyOf(
+ leafJoins)) { // hell. it might remove task from the list. that's sad. I have to copy
+ // it.
+ assert task.isResolved();
+ // a little bit tricky. It assumes that column is join-index backed or just written and
+ // array-backed,
+ advanceAtMinFromEdge(task);
+ }
+ // now let's read each cell's edges, then build "to" side bitset of approximation
+ // first pass: union the contributing pairs' to-doc ranges; every possible match in this
+ // to segment falls into [minToDoc, maxToDoc]
+ for (LeafJoin task : leafJoins) {
+ DocEdges docEdges = task;
+ firstToDoc = Math.min(firstToDoc, docEdges.toDocEdges()[0]);
+ lastToDoc = Math.max(lastToDoc, docEdges.toDocEdges()[1]);
+ matchedToDocsCount += docEdges.toDocEdges()[1] - docEdges.toDocEdges()[0] + 1;
+ if (falsePositiveToDocsBits == null) {
+ falsePositiveToDocsBits = new FixedBitSet(toContext.reader().maxDoc());
+ }
+ falsePositiveToDocsBits.set(docEdges.toDocEdges()[0], docEdges.toDocEdges()[1] + 1);
+ }
+ // when confirming docs, we want to start from heaviest leafs first
+ if (leafJoins.size() > 1) {
+ leafJoins.sort(
+ Comparator.comparingInt(
+ leafJoin ->
+ Math.toIntExact(
+ Math.min(
+ leafJoin
+ .fromMatchCount, // use a min of both side as an estimate of join
+ // cardinality
+ leafJoin.edges.toCount())))
+ .reversed());
+ }
+ logContextSetUp();
+ }
+
+ /**
+ * Reports how this context was set up, once. {@code approxCard} is the real size of the
+ * approximation (the union of the surviving pairs' to-ranges), while {@code approxSpanSum} adds
+ * those ranges up with their overlaps counted twice; the two together say how much the
+ * single-range-per-column approximation actually narrows the segment, which is what bounds every
+ * saving the confirmation phase can make.
+ */
+ private void logContextSetUp() {
+ if (!JoinIndexUtils.diagnosticsEnabled(log)) {
+ return;
+ }
+ long colToCountSum = 0;
+ for (LeafJoin cell : leafJoins) {
+ colToCountSum += cell.toCount();
+ }
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=ctx ctx={} toSeg={} toMaxDoc={} cellsCreated={} cellsDroppedApriori={}"
+ + " cellsLive={} buildMs={} approxCard={} approxSpanSum={} approxFrom={} approxTo={}"
+ + " colToCountSum={}",
+ ctxId,
+ JoinIndexUtils.segmentName(toContext),
+ toContext.reader().maxDoc(),
+ joinLeafsCreated,
+ leafsDroppedApriori,
+ leafJoins.size(),
+ joinIndexBuildNanos / 1_000_000L,
+ falsePositiveToDocsBits == null ? 0 : falsePositiveToDocsBits.cardinality(),
+ matchedToDocsCount,
+ firstToDoc,
+ lastToDoc,
+ colToCountSum);
+ }
+
+ /**
+ * Reports the confirmation totals for this context, once. Emitted when every column has been
+ * drained -- i.e. the half-read union has converged and laziness has run out. A context that
+ * never emits this line never had to converge, which is the case the lazy variant exists for.
+ */
+ private void logConfirmationDone(String reason) {
+ if (reported || !JoinIndexUtils.diagnosticsEnabled(log)) {
+ return;
+ }
+ reported = true;
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=done ctx={} toSeg={} reason={} confirmCalls={} freeHits={} cellsDrained={}"
+ + " cellsLive={} fromDocsWalked={} buildMs={}",
+ ctxId,
+ JoinIndexUtils.segmentName(toContext),
+ reason,
+ confirmCalls,
+ confirmFreeHits,
+ leafsDrained,
+ leafJoins.size(),
+ fromDocsWalked,
+ joinIndexBuildNanos / 1_000_000L);
+ }
+
+ private TaskRefreshResult refreshJoinTasksReferences(IndexSearcher newJoinIndexSearcher)
+ throws IOException, ExecutionException, InterruptedException {
+ Set> joinSegments = new LinkedHashSet<>();
+ Set> justWritten = new LinkedHashSet<>();
+
+ Set refreshReference = new LinkedHashSet<>();
+ Set loadReference = new LinkedHashSet<>();
+ Map needIndex = new LinkedHashMap<>();
+ Set resolveTarget =
+ (newJoinIndexSearcher == this.lastSeenJoinSearcher) ? loadReference : refreshReference;
+ for (LeafJoin task : leafJoins) {
+ if (task.joinSegmentRef != null) {
+ resolveTarget.add(task);
+ } else if (!task.isResolved()) {
+ needIndex.put(task.pairFieldName, task);
+ }
+ // else: resolved from the indexer
+ }
+ // refresh old refs, pass 1: same searcher, just get a segment by ord and check
+ // the segment name
+ List newLeaves = newJoinIndexSearcher.getLeafContexts();
+ for (Iterator iter = refreshReference.iterator(); iter.hasNext(); ) {
+ LeafJoin task = iter.next();
+ // check segment name by ord, if true, resolve it right here, remove from here
+ LeafReaderContext byOrd =
+ task.joinSegmentRef.joinSegmentLeafOrd() < newLeaves.size()
+ ? newLeaves.get(task.joinSegmentRef.joinSegmentLeafOrd())
+ : null;
+ if (byOrd != null
+ && JoinIndexUtils.segmentName(byOrd).equals(task.joinSegmentRef.joinSegmentName())) {
+ joinSegments.add(new SimpleEntry<>(task, byOrd));
+ iter.remove();
+ }
+ }
+ // pass 2: searcher have changed, need to lookup segments by name in the new one
+ if (!refreshReference.isEmpty()) {
+ Map byOldJoinSegName = new HashMap<>();
+ for (LeafJoin task : refreshReference) {
+ byOldJoinSegName.put(task.joinSegmentRef.joinSegmentName(), task);
+ }
+ for (LeafReaderContext joinLeaf : newLeaves) {
+ String segName = JoinIndexUtils.segmentName(joinLeaf);
+ LeafJoin task = byOldJoinSegName.get(segName);
+ if (task != null) {
+ task.joinSegmentRef =
+ new JoinSegmentReference(task.joinSegmentRef.pairFieldName(), segName, joinLeaf.ord);
+ joinSegments.add(new SimpleEntry<>(task, joinLeaf));
+ refreshReference.remove(task);
+ }
+ }
+ }
+ // pass 3: search by field name
+ if (!refreshReference.isEmpty()) {
+ Map byPairFieldName = new HashMap<>();
+ for (LeafJoin task : refreshReference) {
+ byPairFieldName.put(task.joinSegmentRef.pairFieldName(), task);
+ }
+ // loop join segments search for fields
+ Map joinSegmentsByPairFieldName =
+ JoinIndexUtils.extractExistingJoinColumns(
+ lastSeenJoinSearcher, byPairFieldName::containsKey);
+ // if found move to load set
+ for (LeafJoin task : byPairFieldName.values()) {
+ JoinSegmentReference found = joinSegmentsByPairFieldName.get(task.pairFieldName);
+ if (found != null) {
+ task.joinSegmentRef = found;
+ loadReference.add(task);
+ refreshReference.remove(task);
+ }
+ }
+ }
+ if (!refreshReference.isEmpty()) { // TODO presumably we can go to index it
+ throw new IllegalStateException(
+ "unable to refresh segment refs " + refreshReference + " at " + lastSeenJoinSearcher);
+ }
+ // load edges for regulars, repeat pass 1, for those who was found at pass 3
+ for (LeafJoin cell : loadReference) {
+ // String pairFieldName = cell.pairFieldName;
+ LeafReaderContext joinLeafSeg =
+ lastSeenJoinSearcher.getLeafContexts().get(cell.joinSegmentRef.joinSegmentLeafOrd());
+ assert JoinIndexUtils.segmentName(joinLeafSeg).equals(cell.joinSegmentRef.joinSegmentName());
+ joinSegments.add(new SimpleEntry<>(cell, joinLeafSeg));
+ }
+ // index unlucky ones
+ if (!needIndex.isEmpty()) {
+
+ Map missingPairs = new HashMap<>();
+ for (LeafJoin cell : needIndex.values()) {
+ missingPairs.put(cell.pairFieldName, cell.segmentsFromTo);
+ }
+ long buildStartNanos = System.nanoTime();
+ Map written =
+ this.joinIndex.buildAndPersistJoinColumns(
+ Collections.unmodifiableMap(missingPairs),
+ this.fromSearcher.getIndexReader(),
+ this.toReader,
+ this.toField,
+ this.ctxId,
+ // null refs trace back to the weight-age extract, so that's the searcher these
+ // pairs were last seen absent in; still lastSeenJoinSearcher at this point --
+ // it advances to newJoinIndexSearcher only after this method
+ this.lastSeenJoinSearcher,
+ fromColumnFutures);
+ this.joinIndexBuildNanos += System.nanoTime() - buildStartNanos;
+ assert written.keySet().containsAll(missingPairs.keySet());
+ assert missingPairs.keySet().containsAll(written.keySet());
+ for (Map.Entry entry : written.entrySet()) { // TODO optimize
+ LeafJoin cell = needIndex.get(entry.getKey());
+ justWritten.add(new SimpleEntry<>(cell, entry.getValue()));
+ }
+ }
+ this.lastSeenJoinSearcher =
+ newJoinIndexSearcher; // set old searcher, it corresponds to weightAgeJoinSegmentsReadOnly
+ return new TaskRefreshResult(joinSegments, justWritten);
+ }
+
+ /**
+ * Positions {@code cell}'s from-iterator behind {@code docEdges}'s first from-doc, or drops
+ * {@code cell} via {@link #dropJoinLeaf} when the iterator can no longer reach any doc the pair
+ * maps -- either because the pair maps nothing ({-1, -1} sentinel), the iterator already moved
+ * past the pair's last from-doc, or it exhausts before reaching the pair's first one (the
+ * iterator only moves forward, so none of these are recoverable). Returns whether the cell
+ * survives.
+ */
+ private boolean advanceAtMinFromEdge(LeafJoin cell) throws IOException {
+ int[] fromDocEdges = cell.fromDocEdges();
+ int minFromDoc = fromDocEdges[0];
+ int maxFromDoc = fromDocEdges[1];
+ DocIdSetIterator fromSegemtIter = cell.fromSegmentDocIdIter;
+ if (minFromDoc < 0) {
+ // {-1, -1} sentinel: this pair maps no from doc to any to doc at all
+ leafsDroppedApriori++;
+ dropJoinLeaf(cell);
+ return false;
+ }
+ if (maxFromDoc >= 0
+ && maxFromDoc != DocIdSetIterator.NO_MORE_DOCS
+ && fromSegemtIter.docID() > maxFromDoc) {
+ // from iter is already past the last from doc this pair maps, so it cannot contribute
+ leafsDroppedApriori++;
+ dropJoinLeaf(cell); // no more matches in this join segment, so the pair cannot contribute
+ return false;
+ }
+ if (minFromDoc >= 0
+ && maxFromDoc != DocIdSetIterator.NO_MORE_DOCS
+ && fromSegemtIter.docID() < minFromDoc) {
+ int firstMatch = fromSegemtIter.advance(minFromDoc);
+ if (firstMatch == DocIdSetIterator.NO_MORE_DOCS || firstMatch > maxFromDoc) {
+ /// wow from iter exhausted, no match in this join segment, so the pair cannot contribute
+ // thus we need to return them from request `
+ leafsDroppedApriori++;
+ dropJoinLeaf(cell); // no more matches in this join segment, so the pair cannot contribute
+ return false;
+ } // else from iter is advanced behind the first from match , good
+ }
+ return true;
+ }
+
+ /**
+ * every to segment call for prepositioned from seg iters populates a {@link LeafJoin} per
+ * contributing from segment, its iterator PREPOSITIONED to the first matching doc
+ *
+ * @return tasks are orfered by descending from-side match count, so the first task is the one
+ * with the most matches
+ */
+ private List createLeafJoins()
+ throws ExecutionException, InterruptedException, IOException {
+ List tasks = new ArrayList<>();
+ for (Future fromContextFuture : this.fromColumnFutures.values()) {
+ // block on them in the given order, expecting to park on the longest first, and then read
+ // shorter with no locks
+ FromLeafJoinContext matchAndCount = fromContextFuture.get();
+ if (matchAndCount == null) { // no hits for this from-side leaf.
+ continue;
+ }
+ LeafReaderContext fromContext = this.fromSearcher.getLeafContexts().get(matchAndCount.ord);
+ DocIdSetIterator matchedFromDocs;
+ if (matchAndCount.matches != null
+ && (matchedFromDocs = matchAndCount.matches.iterator()) != null
+ && matchedFromDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ // name every contributing (from, to) pair column; pair field names are unique across pairs
+ String pairFieldName =
+ JoinIndexUtils.pairFieldName(fromContext, this.fromField, toContext, this.toField);
+ tasks.add(
+ new LeafJoin(
+ pairFieldName,
+ new SegmentsTuple(fromContext.ord, toContext.ord),
+ matchedFromDocs,
+ matchAndCount.matches.count()));
+ }
+ }
+ return tasks;
+ }
+
+ /** True if this context has any candidate docs at all; used by {@code JoinIndexWeight}. */
+ boolean isEmpty() {
+ return falsePositiveToDocsBits == null || matchedToDocsCount == 0;
+ }
+
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ assert !isEmpty();
+ DocIdSetIterator approximation =
+ new BitSetIterator(falsePositiveToDocsBits, matchedToDocsCount);
+
+ TwoPhaseIterator twoPhase = new LazyConfirmationIterator(approximation);
+ return new ConstantScoreScorer(boost, scoreMode, twoPhase);
+ }
+
+ @Override
+ public long cost() {
+ return (long) lastToDoc - firstToDoc + 1;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java
new file mode 100644
index 000000000000..1d36b8f1c123
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java
@@ -0,0 +1,659 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FilterCodecReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ParallelCompositeReader;
+import org.apache.lucene.index.ParallelLeafReader;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DocIdStream;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.RoaringDocIdSet;
+import org.apache.lucene.util.StringHelper;
+import org.slf4j.Logger;
+import org.slf4j.event.Level;
+
+/**
+ * Column-building and addressing helpers for the auxiliary join index managed by {@link
+ * AuxIndexManager}: for every (from-segment, to-segment) pair it produces a SORTED_NUMERIC column
+ * named {@link #pairFieldName}, whose doc number is the from-side doc id and whose value is the
+ * to-side doc id whose {@code toField} term equals the from doc's {@code fromField} term, plus two
+ * companion edges columns persisting the pair's {min, max} from-doc and to-doc bounds.
+ */
+final class JoinIndexUtils {
+
+ /** Suffix of the always-written column persisting a pair's {min, max} from-doc edges. */
+ static final String FROM_EDGES_PREFIX = "fromDoc_edges_"; // TODO reduce to the singe letter
+
+ /** Suffix of the always-written column persisting a pair's {min, max} to-doc edges. */
+ static final String TO_EDGES_PREFIX = "toDoc_edges_";
+
+ /** main join columns for join index to_doc_num[from_docnum] */
+ static final String TO_DOC_VAL_BY_FROM_DOCNUM = "join_toDoc_";
+
+ static final String TO_COUNT_PREFIX = "num_toDoc_";
+
+ private JoinIndexUtils() {}
+
+ /**
+ * Level the {@code AIJOIN evt=...} diagnostic logs emit at; kept in one place so all diagnostics
+ * can be raised or lowered together. To see them, enable {@code TRACE} for the emitting loggers
+ * in log4j config.
+ */
+ private static final Level LOG_LEVEL = Level.TRACE;
+
+ /** Whether the AIJOIN diagnostic logs would emit at the configured level. */
+ static boolean diagnosticsEnabled(Logger log) {
+ return log.isEnabledForLevel(LOG_LEVEL);
+ }
+
+ /** Emits an AIJOIN diagnostic line at the configured level. */
+ static void logDiagnostic(Logger log, String message, Object... args) {
+ log.atLevel(LOG_LEVEL).log(message, args);
+ }
+
+ /**
+ * Scans {@code joinSearcher}'s leaves for every pair column whose field name satisfies {@code
+ * isNeeded}, returning where each one lives. Used both to seed a fresh {@link JoinIndexWeight}'s
+ * view of already-built pairs, and by {@link JoinIndexScorerSupplier} to relocate a pair whose
+ * cached segment reference no longer resolves. TODO subject for in-heap caching TODO commit's
+ * userdata might have a list of pairs with known segment ords and names
+ */
+ static Map extractExistingJoinColumns(
+ IndexSearcher joinSearcher, Predicate isNeeded) {
+ Map existingJoinSegments =
+ CollectionUtil.newHashMap(joinSearcher.getIndexReader().leaves().size());
+ for (LeafReaderContext joinContext : joinSearcher.getIndexReader().leaves()) {
+ String segmentName = segmentName(joinContext);
+ for (FieldInfo fieldInfo : joinContext.reader().getFieldInfos()) {
+ // pairs are detected by their toCount companion, which is written to doc 0 for every
+ // built pair; the join column itself is sparse and a tombstone pair (disjoint terms)
+ // never materializes it, so scanning for join columns kept re-reporting once-built
+ // tombstones as missing -- and re-triggering their from-side FK loads on every query
+ String splits[] = fieldInfo.name.split(TO_COUNT_PREFIX);
+ if (splits.length == 2 && isNeeded.test(splits[1])) {
+ existingJoinSegments.computeIfAbsent(
+ splits[1],
+ fieldName ->
+ new AuxIndexManager.JoinSegmentReference(
+ fieldName, segmentName, joinContext.ord));
+ }
+ }
+ }
+ return existingJoinSegments;
+ }
+
+ static CacheAndCount computeDocIdSet(Weight fromWeight, LeafReaderContext ctx)
+ throws IOException {
+ ScorerSupplier supplier = fromWeight.scorerSupplier(ctx);
+ if (supplier == null) {
+ return null; // NO matches ???
+ }
+ BulkScorer scorer = supplier.bulkScorer();
+ return cacheImpl(scorer, ctx.reader().maxDoc(), ctx.reader().getLiveDocs());
+ }
+
+ /**
+ * A pair's {min, max} from-doc and to-doc bounds and match count, common to both a pair freshly
+ * built on demand ({@link JoinColumnModel#edges()}) and one already persisted in the join index
+ * ({@link Edges}, loaded through {@link #loadEdges}), so code walking matches doesn't need to
+ * care which one backs it.
+ */
+ interface DocEdges {
+ int[] fromDocEdges();
+
+ int[] toDocEdges();
+
+ /** this is rather doubtful */
+ int toCount();
+ }
+
+ /** A self-contained {@link DocEdges} value, with no addressing information of its own. */
+ record Edges(int[] fromDocEdges, int[] toDocEdges, int toCount) implements DocEdges {}
+
+ /**
+ * The from-doc-to-to-doc map produced by {@link #computeDocMapping}, paired with its resolved
+ * {@link #edges()}. {@link #toDocByFromDoc()} mirrors the on-disk column's read API, so freshly
+ * built pairs (not yet flushed to the join index) and pairs loaded from the join index can be
+ * walked by the same code.
+ */
+ static final class JoinColumnModel {
+ private final int[] toDocByFromDoc;
+ private final DocEdges edges;
+
+ JoinColumnModel(int[] toDocByFromDoc, DocEdges edges) {
+ this.toDocByFromDoc = toDocByFromDoc;
+ this.edges = edges;
+ }
+
+ /**
+ * Returns a fresh single-valued cursor over the from-doc -> to-doc map, positioned before doc
+ * 0.
+ */
+ SortedNumericDocValues toDocByFromDoc() {
+ return new ArrayBackedSortedNumericDocValues(toDocByFromDoc);
+ }
+
+ DocEdges edges() {
+ return edges;
+ }
+
+ /** Returns one greater than the largest possible document number. */
+ public int maxDoc() {
+ return toDocByFromDoc.length;
+ }
+ }
+
+ /**
+ * Adapts an int-array from-doc -> to-doc map (as produced by {@link #computeDocMapping}, {@code
+ * -1} meaning no value) to the {@link SortedNumericDocValues} read API, so it can be consumed the
+ * same way as the on-disk join column. Always single-valued until M:N pairs are supported.
+ */
+ private static final class ArrayBackedSortedNumericDocValues extends SortedNumericDocValues {
+ private final int[] toDocByFromDoc;
+ private int doc = -1;
+
+ ArrayBackedSortedNumericDocValues(int[] toDocByFromDoc) {
+ this.toDocByFromDoc = toDocByFromDoc;
+ }
+
+ @Override
+ public long nextValue() {
+ return toDocByFromDoc[doc];
+ }
+
+ @Override
+ public int docValueCount() {
+ return 1;
+ }
+
+ @Override
+ public boolean advanceExact(int target) {
+ doc = target;
+ return target < toDocByFromDoc.length && toDocByFromDoc[target] >= 0;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public int advance(int target) {
+ while (target < toDocByFromDoc.length && toDocByFromDoc[target] < 0) {
+ target++;
+ }
+ doc = target < toDocByFromDoc.length ? target : NO_MORE_DOCS;
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return toDocByFromDoc.length;
+ }
+ }
+
+ /**
+ * Maps a to-side {@link LeafReaderContext} to an int array of to-side doc ids by ordinal for a
+ * fixed {@code toField}, for those docs that are live. this is quite local lifecycle class, thus
+ * we can cache it so. the trick is that, there always single entry in this cache.
+ */
+ static class ToDocInvertor implements Function {
+ private final String toField;
+ private final Map cache = new HashMap<>();
+
+ ToDocInvertor(String toField) {
+ this.toField = toField;
+ }
+
+ @Override
+ public int[] apply(LeafReaderContext toContext) {
+ return cache.computeIfAbsent(toContext.ord, (i) -> computeToDoc(toContext));
+ }
+
+ private int[] computeToDoc(LeafReaderContext toContext) {
+ try {
+ SortedSetDocValues toDV = DocValues.getSortedSet(toContext.reader(), toField);
+ Bits toLiveDocs = toContext.reader().getLiveDocs();
+ int[] toDocByToOrd = new int[Math.toIntExact(toDV.getValueCount())];
+ Arrays.fill(toDocByToOrd, -1);
+ for (int toDoc = toDV.nextDoc();
+ toDoc != DocIdSetIterator.NO_MORE_DOCS;
+ toDoc = toDV.nextDoc()) {
+ if (toLiveDocs != null && !toLiveDocs.get(toDoc)) {
+ continue;
+ }
+ for (int i = 0; i < toDV.docValueCount(); i++) {
+ toDocByToOrd[(int) toDV.nextOrd()] = toDoc;
+ }
+ }
+ return toDocByToOrd;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ /**
+ * Builds the join column for one (from-segment, to-segment) pair: resolves every from-side doc to
+ * its matching to-side doc id, along with the pair's from-doc and to-doc bounds. From-side terms
+ * are hashed by {@link ForeignKeyColumn}; each to-side term is looked up in that hash to map
+ * from-side ords to to-side ords.
+ *
+ * Docs already deleted at build time are skipped, purely to avoid persisting entries nobody
+ * can ever match -- deletes are otherwise re-checked live at query time (from-side in {@code
+ * JoinIndexScorerSupplier}, to-side by the searcher's own {@code acceptDocs}), since a pair's
+ * cached mapping outlives whatever gets deleted after it was built.
+ */
+ static JoinColumnModel computeDocMapping(
+ LeafReaderContext toContext,
+ String toField,
+ ForeignKeyColumn fromSideData,
+ ToDocInvertor toDocInvertor)
+ throws IOException {
+ assert fromSideData != null;
+
+ long[] toOrdByFromOrd = new long[fromSideData.getFromValuesCount()];
+ Arrays.fill(toOrdByFromOrd, -1L);
+ SortedSetDocValues toDV = DocValues.getSortedSet(toContext.reader(), toField);
+ TermsEnum toTerms = toDV.termsEnum();
+ // resolve from-side ords to to-side ords: look each to-side term up in the from-side hash.
+ boolean termsAreDisjoint = true;
+ for (BytesRef term = toTerms.next(); term != null; term = toTerms.next()) {
+ int fromOrd = fromSideData.getFromTermOrdOrDashOne(term);
+ if (fromOrd != -1) {
+ toOrdByFromOrd[fromOrd] = (int) toTerms.ord();
+ termsAreDisjoint = false;
+ }
+ }
+ // TODO: this degrades M:N joins to M:1. Both toDocByToOrd and toDocByFromDoc keep a single
+ // to-side doc per slot, so when several to docs share a term (non-unique toField) or a
+ // fromSideData
+ // doc is multi-valued with several matching terms, later assignments overwrite earlier ones
+ // and only the last match survives. The read side (AuxIndexJoinQuery) already consumes all
+ // docValueCount() values per doc, so only this writer needs to learn to emit multiple
+ // to docs per fromSideData doc.
+ if (!termsAreDisjoint) {
+ int[] toDocByToOrd = toDocInvertor.apply(toContext);
+
+ // resolve every fromSideData doc to its to-side doc. Docs without the field, or whose term
+ // has
+ // no to-side match, keep -1.
+ int[] toDocByFromDoc = fromSideData.cloneFromOrdByFromDoc();
+ int minFromDoc = DocIdSetIterator.NO_MORE_DOCS;
+ int maxFromDoc = -1;
+ int minToDoc = DocIdSetIterator.NO_MORE_DOCS;
+ int maxToDoc = -1;
+ int toCount = 0;
+ // walk the array, mapping each fromSideData ord to its to-side doc in place.
+ for (int fromDoc = 0; fromDoc < toDocByFromDoc.length; fromDoc++) {
+ int fromOrd = toDocByFromDoc[fromDoc];
+ if (fromOrd == -1) {
+ continue;
+ }
+ int toOrd = (int) toOrdByFromOrd[fromOrd];
+ int toDoc = toOrd == -1 ? -1 : toDocByToOrd[toOrd];
+ if (toDoc == -1) {
+ toDocByFromDoc[fromDoc] = -1; // wiping is crucial
+ continue;
+ }
+ toDocByFromDoc[fromDoc] = toDoc;
+ minFromDoc = Math.min(minFromDoc, fromDoc);
+ maxFromDoc = Math.max(maxFromDoc, fromDoc);
+ minToDoc = Math.min(minToDoc, toDoc);
+ maxToDoc = Math.max(maxToDoc, toDoc);
+ toCount++;
+ }
+ if (maxFromDoc < 0) { // tombstone - column is empty
+ // no fromSideData doc in this pair maps to any to doc: normalize both edges to the
+ // symmetric
+ // {-1, -1} sentinel. An asymmetric one (e.g. {NO_MORE_DOCS, -1}) doesn't round-trip
+ // through the join index's SORTED_NUMERIC edges column, which always returns its two
+ // values in ascending numeric order regardless of which was written as "min" -- so
+ // {NO_MORE_DOCS, -1} silently comes back as {-1, NO_MORE_DOCS} on the next read.
+ minFromDoc = -1;
+ minToDoc = -1;
+ maxToDoc = -1;
+ }
+ return new JoinColumnModel(
+ toDocByFromDoc,
+ new Edges(new int[] {minFromDoc, maxFromDoc}, new int[] {minToDoc, maxToDoc}, toCount));
+ } else { // tombstone - column is empty, due to disjoint terms, perhaps one may optimize it
+ int[] minusones = new int[fromSideData.fromSideMaxDocs()];
+ Arrays.fill(minusones, -1);
+ return new JoinColumnModel(minusones, new Edges(new int[] {-1, -1}, new int[] {-1, -1}, 0));
+ }
+ }
+
+ /**
+ * Reads a pair's persisted {@code {min, max}} edges (or {@code toCount}), all stored on doc 0 of
+ * the column -- the read-side counterpart of {@link JoinColumWriter}'s edges columns.
+ */
+ static int[] loadEdges(LeafReaderContext joinContext, String edgesFieldName) throws IOException {
+ SortedNumericDocValues edgesDV = joinContext.reader().getSortedNumericDocValues(edgesFieldName);
+ assert edgesDV != null : "expected edges column to be present: " + edgesFieldName;
+ int zeroDoc = edgesDV.nextDoc();
+ assert zeroDoc == 0 : "expected edges column to be fully materialized, but got doc " + zeroDoc;
+ int[] values = new int[edgesDV.docValueCount()];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = (int) edgesDV.nextValue();
+ }
+ return values;
+ }
+
+ /**
+ * The join index field name addressing the ordinal map of one (from-segment, to-segment) pair.
+ */
+ static String pairFieldName(
+ LeafReaderContext fromContext,
+ String fromField,
+ LeafReaderContext toContext,
+ String toField) {
+ return getSideKey(fromContext, fromField) + "_" + getSideKey(toContext, toField);
+ }
+
+ /**
+ * Persistent identifier of one join side: the join field name, the immutable id the segment was
+ * created with (it survives reopens, growing deletes mask and reorderings of {@link
+ * IndexReader#leaves()}; a merge produces a new segment with a new id) and the docvalues
+ * generation of the join field.
+ */
+ static String getSideKey(LeafReaderContext context, String field) {
+ byte[] segmentId = segmentReader(context.reader()).getSegmentInfo().info.getId();
+ // dvGen starts at -1 and advances only when this particular field receives an in-place
+ // IndexWriter.updateDocValues update; deletes only bump delGen and leave it untouched. So the
+ // key is insensitive to deletes but changes when the join field's docvalues are updated.
+ FieldInfo fieldInfo = context.reader().getFieldInfos().fieldInfo(field);
+ long dvGen = fieldInfo == null ? -1 : fieldInfo.getDocValuesGen();
+ String key = field + ":" + StringHelper.idToString(segmentId) + ":" + dvGen;
+ return key;
+ }
+
+ /**
+ * Peels wrappers off a leaf reader down to its {@link SegmentReader}. {@link
+ * FilterLeafReader#unwrap} alone is not enough: wrappers like {@code
+ * SoftDeletesDirectoryReaderWrapper} produce {@link FilterCodecReader} leaves, which are not
+ * {@link FilterLeafReader}s, and the two kinds may alternate. Tests additionally wrap readers in
+ * a {@link ParallelLeafReader} (e.g. {@code LuceneTestCase#newSearcher}); that class can in
+ * general combine several independent readers side by side, but when it wraps exactly one (the
+ * common case, including every reader the test framework wraps purely for coverage) that one is
+ * unambiguous and safe to descend into.
+ */
+ static SegmentReader segmentReader(LeafReader reader) {
+ while (true) {
+ if (reader instanceof SegmentReader segmentReader) {
+ return segmentReader;
+ } else if (reader instanceof FilterLeafReader filterLeafReader) {
+ reader = filterLeafReader.getDelegate();
+ } else if (reader instanceof FilterCodecReader filterCodecReader) {
+ reader = filterCodecReader.getDelegate();
+ } else if (reader instanceof ParallelLeafReader parallelLeafReader) {
+ LeafReader[] parallelReaders = parallelLeafReader.getParallelReaders();
+ if (parallelReaders.length != 1) {
+ throw new IllegalArgumentException(
+ "cannot unwrap a SegmentReader from a ParallelLeafReader combining "
+ + parallelReaders.length
+ + " independent readers");
+ }
+ reader = parallelReaders[0];
+ } else {
+ throw new IllegalArgumentException(
+ "cannot unwrap a SegmentReader from " + reader.getClass().getName());
+ }
+ }
+ }
+
+ /** The name of the sidecar segment carrying the given join index leaf. */
+ static String segmentName(LeafReaderContext joinContext) {
+ return segmentReader(joinContext.reader()).getSegmentName();
+ }
+
+ /**
+ * The {@link Directory} backing {@code reader}, unwrapped from any composite that lacks a single
+ * directory of its own (e.g. a {@link ParallelCompositeReader}): a {@link DirectoryReader}
+ * exposes its directory directly, anything else contributes the first leaf-reader whose directory
+ * owns it (descending through {@link ParallelLeafReader} and codec/filter wrappers).
+ */
+ static Directory directory(IndexReader reader) {
+ if (reader instanceof DirectoryReader dr) {
+ return dr.directory();
+ }
+ for (LeafReaderContext leaf : reader.leaves()) {
+ return segmentReader(leaf.reader()).getSegmentInfo().info.dir;
+ }
+ throw new IllegalArgumentException("no directory backing " + reader);
+ }
+
+ /**
+ * A hashable key identifying the storage location behind {@code directory}, stable across
+ * separate opens of the same path so repeated calls resolve to the same cache entry.
+ */
+ static Object directoryKey(Directory directory) throws IOException {
+ Directory unwrapped = FilterDirectory.unwrap(directory);
+ if (unwrapped instanceof FSDirectory fsDir) {
+ return fsDir.getDirectory().toRealPath(); // Path has proper equals/hashCode
+ }
+ return unwrapped; // RAMDirectory/ByteBuffersDirectory etc: identity is the real key
+ }
+
+ /**
+ * Every pair field name (the part after {@link #TO_COUNT_PREFIX}) present in {@code fieldInfos},
+ * i.e. every join pair this segment carries, needed or not. Detected by the toCount companion --
+ * written for every built pair -- not by the join column itself, which a tombstone pair (disjoint
+ * terms) never materializes; the reaper must see tombstones too, or it would judge a segment
+ * carrying only tombstones as pair-free and reap it, resurrecting the pairs' rebuilds.
+ */
+ static Set pairFieldNames(FieldInfos fieldInfos) {
+ Set names = new HashSet<>();
+ for (FieldInfo fieldInfo : fieldInfos) {
+ String[] splits = fieldInfo.name.split(TO_COUNT_PREFIX);
+ if (splits.length == 2) {
+ names.add(splits[1]);
+ }
+ }
+ return names;
+ }
+
+ /**
+ * Reads a segment's {@link FieldInfos} straight off disk, without opening a full reader. Mirrors
+ * {@code IndexWriter#readFieldInfos}, which isn't visible outside its package.
+ */
+ static FieldInfos readFieldInfos(SegmentCommitInfo info) throws IOException {
+ Codec codec = info.info.getCodec();
+ FieldInfosFormat fieldInfosFormat = codec.fieldInfosFormat();
+ if (info.hasFieldUpdates()) {
+ String segmentSuffix = Long.toString(info.getFieldInfosGen(), Character.MAX_RADIX);
+ return fieldInfosFormat.read(info.info.dir, info.info, segmentSuffix, IOContext.READONCE);
+ } else if (info.info.getUseCompoundFile()) {
+ try (Directory cfs = codec.compoundFormat().getCompoundReader(info.info.dir, info.info)) {
+ return fieldInfosFormat.read(cfs, info.info, "", IOContext.READONCE);
+ }
+ } else {
+ return fieldInfosFormat.read(info.info.dir, info.info, "", IOContext.READONCE);
+ }
+ }
+
+ // copy of org.apache.lucene.search.LRUQueryCache.cacheIntoRoaringDocIdSet
+ protected static class CacheAndCount implements Accountable {
+ protected static final CacheAndCount EMPTY = new CacheAndCount(DocIdSet.EMPTY, 0);
+
+ private static final long BASE_RAM_BYTES_USED =
+ RamUsageEstimator.shallowSizeOfInstance(CacheAndCount.class);
+ private final DocIdSet cache;
+ private final int count;
+
+ public CacheAndCount(DocIdSet cache, int count) {
+ this.cache = cache;
+ this.count = count;
+ }
+
+ public DocIdSetIterator iterator() throws IOException {
+ return cache.iterator();
+ }
+
+ // TODO is it really a cost or a count???
+ public int count() {
+ return count;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return BASE_RAM_BYTES_USED + cache.ramBytesUsed();
+ }
+ }
+
+ static CacheAndCount cacheImpl(BulkScorer scorer, int maxDoc, Bits liveDocs) throws IOException {
+ if (scorer.cost() * 100 >= maxDoc) {
+ // FixedBitSet is faster for dense sets and will enable the random-access
+ // optimization in ConjunctionDISI
+ return cacheIntoBitSet(scorer, maxDoc, liveDocs);
+ } else {
+ return cacheIntoRoaringDocIdSet(scorer, maxDoc, liveDocs);
+ }
+ }
+
+ private static CacheAndCount cacheIntoBitSet(BulkScorer scorer, int maxDoc, Bits liveDocs)
+ throws IOException {
+ final FixedBitSet bitSet = new FixedBitSet(maxDoc);
+ int[] count = new int[1];
+ scorer.score(
+ new LeafCollector() {
+
+ private int[] buffer;
+
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {}
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (liveDocs == null || liveDocs.get(doc)) {
+ count[0]++;
+ bitSet.set(doc);
+ }
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ if (buffer == null) {
+ buffer = new int[128];
+ }
+ for (int c = stream.intoArray(buffer); c != 0; c = stream.intoArray(buffer)) {
+ int skip = 0;
+ for (int i = 0; i < c; ++i) {
+ if (liveDocs != null && !liveDocs.get(buffer[i])) {
+ skip++;
+ continue;
+ }
+ bitSet.set(buffer[i]);
+ }
+ count[0] += c - skip;
+ }
+ }
+ },
+ null,
+ 0,
+ DocIdSetIterator.NO_MORE_DOCS);
+ return new CacheAndCount(new BitDocIdSet(bitSet, count[0]), count[0]);
+ }
+
+ private static CacheAndCount cacheIntoRoaringDocIdSet(
+ BulkScorer scorer, int maxDoc, Bits liveDocs) throws IOException {
+ RoaringDocIdSet.Builder builder = new RoaringDocIdSet.Builder(maxDoc);
+ scorer.score(
+ new LeafCollector() {
+
+ private int[] buffer = null;
+
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {}
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (liveDocs != null && !liveDocs.get(doc)) {
+ return;
+ }
+ builder.add(doc);
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ if (buffer == null) {
+ buffer = new int[128];
+ }
+ for (int c = stream.intoArray(buffer); c != 0; c = stream.intoArray(buffer)) {
+ for (int i = 0; i < c; ++i) {
+ if (liveDocs != null && !liveDocs.get(buffer[i])) {
+ continue;
+ }
+ builder.add(buffer[i]);
+ }
+ }
+ }
+ },
+ null,
+ 0,
+ DocIdSetIterator.NO_MORE_DOCS);
+ RoaringDocIdSet cache = builder.build();
+ return new CacheAndCount(cache, cache.cardinality());
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexWeight.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexWeight.java
new file mode 100644
index 000000000000..f3eb38cac034
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexWeight.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.JoinSegmentReference;
+
+final class JoinIndexWeight extends Weight {
+ final IndexSearcher maybeStaleJoinSearcher;
+ final Map existingJoinSegments;
+ private final ScoreMode scoreMode;
+ private final float boost;
+ private final IndexReader toReader;
+ private final Map> fromColumnFutures;
+
+ /**
+ * [toSegmentOrd][fromSegmentOrd] -> the pair column resolved at construction time; null where the
+ * from segment has no cached matches, the pair maps no from-side values, or no cached match falls
+ * into the pair's from-doc range. Pair columns record the sidecar segment name, not a leaf
+ * context, so they stay valid across join reader refreshes.
+ */
+ // private final PairColumn[][] pairColumnsByTo;
+
+ JoinIndexWeight(
+ AuxIndexJoinQuery auxIndexJoinQuery,
+ IndexSearcher maybeStaleJoinSearcher,
+ Map existingJoinSegments,
+ IndexReader toReader,
+ ScoreMode scoreMode,
+ float boost,
+ Map> foreignColsFutures) {
+ super(auxIndexJoinQuery);
+ this.maybeStaleJoinSearcher = maybeStaleJoinSearcher;
+ this.existingJoinSegments = existingJoinSegments;
+ this.scoreMode = scoreMode;
+ this.boost = boost;
+ this.toReader = toReader;
+ this.fromColumnFutures = foreignColsFutures;
+ }
+
+ private AuxIndexJoinQuery aiJQuery() {
+ return (AuxIndexJoinQuery) this.parentQuery;
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ ScorerSupplier supplier = scorerSupplier(context);
+ if (supplier != null) {
+ Scorer scorer = supplier.get(1);
+ if (scorer.iterator().advance(doc) == doc) {
+ return Explanation.match(scorer.score(), aiJQuery().toString());
+ }
+ }
+ return Explanation.noMatch(aiJQuery().toString());
+ }
+
+ @Override
+ public int count(LeafReaderContext context) throws IOException {
+ // cost() is a range-size upper bound, not an exact match count, so counting has to
+ // fall back to actually driving the two-phase iterator
+ return super.count(context);
+ }
+
+ @Override
+ public ScorerSupplier scorerSupplier(LeafReaderContext tolrc) throws IOException {
+ IndexSearcher joinSearcher = aiJQuery().joinIndex.acquire();
+ try {
+ JoinIndexScorerSupplier ctx = null;
+ try {
+ ctx =
+ new JoinIndexScorerSupplier(
+ tolrc,
+ aiJQuery().fromField,
+ aiJQuery().fromQuery,
+ aiJQuery().fromSearcher,
+ aiJQuery().toField,
+ this.toReader,
+ this.existingJoinSegments,
+ this.maybeStaleJoinSearcher,
+ joinSearcher,
+ aiJQuery().joinIndex,
+ this.fromColumnFutures,
+ scoreMode,
+ boost);
+ } catch (ExecutionException e) { // TODO review exception
+ throw new RuntimeException(e);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return ctx.isEmpty() ? null : ctx;
+ } finally {
+ aiJQuery().joinIndex.release(joinSearcher);
+ }
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext lrc) {
+ // matches depend on the from-side searcher and the external join index, which the
+ // query cache cannot see
+ return false;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/package-info.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/package-info.java
new file mode 100644
index 000000000000..191f99d6c999
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/package-info.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ *
+ * Auxiliary Index Join
+ *
+ * Persists doc-nums to doc-nums relation in auxiliary index.
+ *
+ * Simplified Collaboration Diagram
+ *
+ *
+ * {@link org.apache.solr.search.join.AuxIndexJoinQParserPlugin}
+ * │
+ * ▼
+ * {@link org.apache.solr.search.join.auxindexjoin.AuxIndexManager} ──────▶ {@link org.apache.solr.search.join.auxindexjoin.AuxIndexJoinConfig}
+ * │ │
+ * │ └──────────▶ {@link org.apache.solr.search.join.auxindexjoin.AuxIndexJoinMergePolicy}
+ * ▼
+ * {@link org.apache.solr.search.join.auxindexjoin.AuxIndexJoinQuery} ──────▶ {@link org.apache.solr.search.join.auxindexjoin.FromLeafJoinContext}
+ * │ │
+ * ▼ ▼
+ * {@link org.apache.solr.search.join.auxindexjoin.JoinIndexWeight} {@link org.apache.solr.search.join.auxindexjoin.ForeignKeyColumn}
+ * │
+ * ▼
+ * {@link org.apache.solr.search.join.auxindexjoin.JoinIndexScorerSupplier} ──────▶ {@link org.apache.solr.search.join.auxindexjoin.JoinColumnIndexer}
+ * │ │ │ │
+ * │ ▼ │ ▼
+ * │{@link org.apache.solr.search.join.auxindexjoin.JoinIndexScorerSupplier.LeafJoin} │{@link org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel}
+ * ▼ ▼
+ * {@link org.apache.solr.search.join.auxindexjoin.JoinIndexScorerSupplier.LazyConfirmationIterator} {@link org.apache.solr.search.join.auxindexjoin.JoinColumnDocWriter}
+ *
+ *
+ *
+ * {@link org.apache.solr.search.join.auxindexjoin.AuxIndexManager} owns an auxiliary Lucene
+ * index persisting, for every (from-segment, to-segment) pair, a SORTED_NUMERIC column mapping
+ * from-side doc ids to to-side doc ids, plus edges columns with the pair's {@code {min, max}} doc
+ * bounds and a to-side count. Pair columns are named by both sides' persistent side keys (segment
+ * id + docvalues generation of the join field), so they survive reopens of either side and are
+ * built lazily — the first {@code AuxIndexJoinQuery} weight that needs a pair writes it. See {@link
+ * org.apache.solr.search.join.auxindexjoin.AuxIndexManager} for the user-facing API and {@link
+ * org.apache.solr.search.join.auxindexjoin.AuxIndexJoinConfig} for tunables (blocking vs.
+ * non-blocking refresh, one-field-per-segment, the reaper's sweep interval).
+ *
+ *
A batch of pair columns is written through {@code JoinColumnDocWriter}. It guarantes a batch
+ * lands doc-for-doc in one sidecar segment, so doc 0's edges and every from-doc id line up the same
+ * way.
+ *
+ *
Garbage collection of dead pairs
+ *
+ * The sidecar is append-only. A side key dies when its segment is merged away, dropped, or its join
+ * field receives an in-place docvalues update (dvGen bump); pair columns referencing a dead side
+ * key can never be read again. {@code AuxIndexJoinMergePolicy} reaps them:
+ *
+ *
+ * - Death signal via sampling, not listeners. {@code AuxIndexManager#onCreateWeight}
+ * (called from {@code AuxIndexJoinQuery#createWeight}) reports the set of pair field names a
+ * query actually needed to {@code AuxIndexJoinMergePolicy#onCreateWeight}, keyed by
+ * (from-directory, to-directory). A pair field name present in an earlier snapshot for the
+ * same searcher pair but missing from a later one is queued as a pending removal. Sampling is
+ * throttled ({@code AuxIndexJoinConfig#setSweepSamplingInterval}, default one minute) since
+ * it is only a heuristic hint, not a correctness requirement, and both the snapshot map and
+ * the pending-removal set are size-bounded (best-effort LRU-ish eviction).
+ *
- Reaping. {@code findMerges} drops every sidecar segment whose pair field names are
+ * all pending removals, via a {@code OneMerge} that reports the segment as fully deleted
+ * ({@code wrapForMerge} returns a {@code MatchNoBits} live-docs view) so {@code IndexWriter}
+ * discards it instead of rewriting it. This runs piggybacked on ordinary merges — no
+ * background thread. {@code TestAIJoinMergePolicy} and {@code droppedSegmentCount()} / {@code
+ * pendingPairRemovalsCount()} cover this end to end.
+ *
- Known gaps. A pair field name is only queued for removal once a searcher pair that
+ * used to need it is sampled again without needing it — a side key that dies without
+ * ever being resampled this way stays live.
+ *
+ *
+ * TODO
+ *
+ *
+ * - comparative benchmarking for updates: parent, children field, PARENT_ID_FK ({@code
+ * AIJoinBenchmark} currently only compares against {@code JoinUtil} on a static index)
+ *
- many-to-many: {@code JoinIndexUtils} doc mapping currently degrades M:N joins to M:1
+ * (single to-doc kept per from-doc)
+ *
- reverse join on the same columns: children by parents filter
+ *
- we estimate to&from set by a range. Alternatives: union of ranges, roaring (bitset).
+ * Note: to and from estimates might be built different. For "to" side we need to provide
+ * advance()-eble iterator over union. And for "from" side it should be just intersectable
+ * with docSetIter. Format should balance storage size and decoding efforts. It should be
+ * stored as a Document fields.
+ *
- stripe columns for join index: break {@code JoinIndexUtils.TO_DOC_VAL_BY_FROM_DOCNUM} to a
+ * pair one is {@code to_doc_nums=maxdocs(to-side)/2}
+ *
+ */
+package org.apache.solr.search.join.auxindexjoin;
diff --git a/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/schema.xml b/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/schema.xml
new file mode 100644
index 000000000000..f465bb48d5fe
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/schema.xml
@@ -0,0 +1,31 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ id
+
diff --git a/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/solrconfig.xml
new file mode 100644
index 000000000000..d30c890c7b2a
--- /dev/null
+++ b/solr/core/src/test-files/solr/configsets/auxindexjoin/conf/solrconfig.xml
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+ ${tests.luceneMatchVersion:LATEST}
+
+ ${solr.data.dir:}
+
+
+
+
+
+
diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribAuxIndexJoinFromCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribAuxIndexJoinFromCollectionTest.java
new file mode 100644
index 000000000000..0f3f49ef6eb5
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DistribAuxIndexJoinFromCollectionTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import static org.apache.solr.cloud.MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.solr.client.solrj.RemoteSolrException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.search.join.AuxIndexJoinQParserPlugin;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link AuxIndexJoinQParserPlugin} version of {@link DistribJoinFromCollectionTest}: same
+ * collocated-replica {@code fromIndex} setup (a single-shard "from" collection deployed onto every
+ * node holding a "to" replica, so {@link
+ * org.apache.solr.search.join.ScoreJoinQParserPlugin#getCoreName}'s collocation lookup always finds
+ * a local "from" replica), but adapted for two ways {@code {!auxIndexJoin}} differs from {@code
+ * {!join}}:
+ *
+ *
+ * - no scoring -- {@code {!auxIndexJoin}} is always a constant-score match, so there's no
+ * {@code score=} local param and no scoring assertions.
+ *
- M:1 only -- {@code JoinIndexUtil#computeDocMapping} keeps exactly one to-doc per from-doc,
+ * so the join here always goes from the "from" collection's docs (each with a single-valued
+ * FK) to the "to" collection's docs by their unique key, never the reverse (one "to" doc
+ * resolving to many "from" docs would silently drop matches).
+ *
+ */
+public class DistribAuxIndexJoinFromCollectionTest extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String toColl = "aijoin_to_2x2";
+ private static final String fromColl = "aijoin_from_1x4";
+
+ private static String toDocId;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ String configName = "aijoinCloudCollectionConfig";
+ int nodeCount = 5;
+ configureCluster(nodeCount)
+ .withSolrXml(
+ DEFAULT_CLOUD_SOLR_XML.replace(
+ "", "4"))
+ .addConfig(configName, configset("auxindexjoin"))
+ .configure();
+
+ Map collectionProperties = new HashMap<>();
+
+ // create a collection holding data for the "to" (unique-key) side of the JOIN
+ int shards = 2;
+ int replicas = 2;
+ CollectionAdminRequest.createCollection(toColl, configName, shards, replicas)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ // get the set of nodes where replicas for the "to" collection exist
+ Set nodeSet = new HashSet<>();
+ ZkStateReader zkStateReader = cluster.getZkStateReader();
+ ClusterState cs = zkStateReader.getClusterState();
+ for (Slice slice : cs.getCollection(toColl).getActiveSlices())
+ for (Replica replica : slice.getReplicas()) nodeSet.add(replica.getNodeName());
+ assertTrue(nodeSet.size() > 0);
+
+ // deploy the single-shard "from" collection to every node where the "to" collection exists,
+ // so AuxIndexJoinQParserPlugin's collocation lookup always finds a local "from" replica -- and
+ // stays
+ // on the single-shard fast path, which skips the router-field cross-checks that a sharded
+ // "from" collection would otherwise require
+ CollectionAdminRequest.createCollection(fromColl, configName, 1, 4)
+ .setCreateNodeSet(String.join(",", nodeSet))
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ toDocId = indexToDoc(1001, "b");
+ indexFromDoc(2001, toDocId, "c");
+ indexFromDoc(2002, toDocId, "d"); // a second "from" doc mapping to the same "to" doc: still M:1
+
+ Thread.sleep(1000); // so the commits fire
+ }
+
+ @AfterClass
+ public static void shutdown() {
+ log.info(
+ "DistribAuxIndexJoinFromCollectionTest logic complete ... deleting the {} and {} collections",
+ toColl,
+ fromColl);
+
+ // try to clean up
+ for (String c : new String[] {toColl, fromColl}) {
+ try {
+ CollectionAdminRequest.Delete req = CollectionAdminRequest.deleteCollection(c);
+ req.process(cluster.getSolrClient());
+ } catch (Exception e) {
+ // don't fail the test
+ log.warn("Could not delete collection {} after test completed due to:", c, e);
+ }
+ }
+
+ log.info("DistribAuxIndexJoinFromCollectionTest succeeded ... shutting down now!");
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ // verify the join with fromIndex works: two different "from" docs (match_s:c and match_s:d)
+ // both resolve to the same single "to" doc, still M:1 since it's one to-doc per from-doc
+ CloudSolrClient client = cluster.getSolrClient();
+ {
+ final String joinQ = "{!auxIndexJoin from=join_s fromIndex=" + fromColl + " to=id}match_s:c";
+ QueryRequest qr =
+ new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
+ QueryResponse rsp = qr.process(client);
+ SolrDocumentList hits = rsp.getResults();
+ assertEquals("Expected 1 doc, got " + hits, 1, hits.getNumFound());
+ SolrDocument doc = hits.get(0);
+ assertEquals(toDocId, doc.getFirstValue("id"));
+ assertEquals("b", doc.getFirstValue("get_s"));
+ // {!auxIndexJoin} never scores -- always a constant-score match, like {!join score=none}
+ assertEquals("1.0", doc.getFirstValue("score").toString());
+ }
+
+ // negative test before creating an alias
+ checkAbsentFromIndex();
+
+ // create an alias for the fromIndex and then query through the alias
+ String alias = fromColl + "Alias";
+ CollectionAdminRequest.createAlias(alias, fromColl).process(client);
+
+ {
+ final String joinQ = "{!auxIndexJoin from=join_s fromIndex=" + alias + " to=id}match_s:d";
+ final QueryRequest qr =
+ new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
+ final QueryResponse rsp = qr.process(client);
+ final SolrDocumentList hits = rsp.getResults();
+ assertEquals("Expected 1 doc", 1, hits.getNumFound());
+ SolrDocument doc = hits.get(0);
+ assertEquals(toDocId, doc.getFirstValue("id"));
+ assertEquals("b", doc.getFirstValue("get_s"));
+ }
+
+ // negative test after creating an alias
+ checkAbsentFromIndex();
+
+ {
+ // verify join doesn't work if no match in the "from" index
+ final String joinQ =
+ "{!auxIndexJoin from=join_s fromIndex=" + fromColl + " to=id}match_s:nomatch";
+ final QueryRequest qr =
+ new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
+ final QueryResponse rsp = qr.process(client);
+ final SolrDocumentList hits = rsp.getResults();
+ assertEquals("Expected no hits", 0, hits.getNumFound());
+ }
+ }
+
+ private void checkAbsentFromIndex() {
+ final String wrongName = fromColl + "WrongName";
+ final String joinQ = "{!auxIndexJoin from=join_s fromIndex=" + wrongName + " to=id}match_s:c";
+ final QueryRequest qr =
+ new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
+ RemoteSolrException ex =
+ assertThrows(RemoteSolrException.class, () -> cluster.getSolrClient().request(qr));
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code());
+ assertTrue(ex.getMessage().contains(wrongName));
+ }
+
+ /** Indexes a "to"-side doc: {@code id} is the join target, unique per doc (required for M:1). */
+ private static String indexToDoc(int id, String getField) throws Exception {
+ UpdateRequest up = new UpdateRequest();
+ up.setCommitWithin(50);
+ up.setParam("collection", toColl);
+ SolrInputDocument doc = new SolrInputDocument();
+ String docId = "" + id;
+ doc.addField("id", docId);
+ doc.addField("get_s", getField);
+ up.add(doc);
+ cluster.getSolrClient().request(up);
+ return docId;
+ }
+
+ /** Indexes a "from"-side doc with a single-valued {@code join_s} FK pointing to a "to" doc. */
+ private static void indexFromDoc(int id, String joinFieldValue, String matchField)
+ throws Exception {
+ UpdateRequest up = new UpdateRequest();
+ up.setCommitWithin(50);
+ up.setParam("collection", fromColl);
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "" + id);
+ doc.addField("join_s", joinFieldValue);
+ doc.addField("match_s", matchField);
+ up.add(doc);
+ cluster.getSolrClient().request(up);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/join/TestAuxIndexJoinQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/join/TestAuxIndexJoinQParserPlugin.java
new file mode 100644
index 000000000000..d5f0835b30f2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/TestAuxIndexJoinQParserPlugin.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join;
+
+import static org.apache.solr.SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME;
+import static org.apache.solr.SolrTestCaseJ4.assertFieldValues;
+import static org.apache.solr.SolrTestCaseJ4.configset;
+import static org.apache.solr.SolrTestCaseJ4.params;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.commons.io.file.PathUtils;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.cloud.DistribAuxIndexJoinFromCollectionTest;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.join.auxindexjoin.AuxIndexManager;
+import org.apache.solr.util.EmbeddedSolrServerTestRule;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+/**
+ * Exercises {@link AuxIndexJoinQParserPlugin}, modeled on {@link TestScoreJoinQPNoScore}'s
+ * same-core {@code {!join}} coverage plus {@link org.apache.solr.TestCrossCoreJoin}'s cross-core
+ * setup, since cross-core joins -- not same-core -- are the reason {@link AuxIndexManager} exists.
+ *
+ * Every join here goes from the "many" side (an employee, single-valued FK) to the "few" side
+ * (that employee's department, a value unique per to-doc): {@code AIJoinUtil#computeDocMapping} is,
+ * per its own javadoc, "always single-valued until M:N pairs are supported" -- it keeps exactly one
+ * to-doc per from-doc. That's exact for this direction (each employee has exactly one department),
+ * but would silently drop matches for the reverse, one-department-to-many-employees direction, so
+ * this test doesn't exercise that one.
+ *
+ *
Both cores use version 1.7 schemas ({@code configsets/aijoin}, shared with {@link
+ * DistribAuxIndexJoinFromCollectionTest}, and {@code configsets/minimal}), where docValues default
+ * to true: {@link AuxIndexManager} reads real per-segment {@link
+ * org.apache.lucene.index.SortedSetDocValues} directly, unlike {@link ScoreJoinQParserPlugin} (via
+ * {@link org.apache.lucene.search.join.JoinUtil}) which tolerates uninverted fields too.
+ */
+public class TestAuxIndexJoinQParserPlugin extends SolrTestCase {
+
+ private static final String FROM_CORE = "aijoinFromCore";
+
+ @ClassRule
+ public static final EmbeddedSolrServerTestRule solrRule = new EmbeddedSolrServerTestRule();
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ solrRule.startSolr();
+
+ // the aijoin configset declares no schemaFactory, so loading it file-based would let the
+ // default managed-schema factory rewrite the source-tree conf dir; work on a temp copy
+ Path aijoinConfigSet = createTempDir("auxindexjoin");
+ PathUtils.copyDirectory(configset("auxindexjoin"), aijoinConfigSet.resolve("conf"));
+ solrRule.newCollection().withConfigSet(aijoinConfigSet).create();
+
+ // "minimal" uses ClassicIndexSchemaFactory, which never writes, so it's safe to use in place
+ solrRule.newCollection(FROM_CORE).withConfigSet(configset("minimal")).create();
+
+ SolrClient toSide = solrRule.getSolrClient();
+
+ // departments: the "to"/unique side of every join below, one doc per dept_id_s value
+ toSide.add(
+ List.of(
+ doc("id", "10", "dept_id_s", "Engineering", "text_s", "These guys develop stuff"),
+ doc("id", "11", "dept_id_s", "Marketing", "text_s", "These guys make you look good"),
+ doc("id", "12", "dept_id_s", "Sales", "text_s", "These guys sell stuff"),
+ doc("id", "13", "dept_id_s", "Support", "text_s", "These guys help customers")));
+
+ // employees: the "from"/many side, each with exactly one dept_s value
+ toSide.add(employees());
+ toSide.commit();
+
+ // aijoinFromCore holds the same employees, for the cross-core variant of the same joins
+ SolrClient fromSide = solrRule.getSolrClient(FROM_CORE);
+ fromSide.add(employees());
+ fromSide.commit();
+ }
+
+ private static List employees() {
+ return List.of(
+ doc("id", "1", "name_s", "john", "title_s", "Director", "dept_s", "Engineering"),
+ doc("id", "2", "name_s", "mark", "title_s", "VP", "dept_s", "Marketing"),
+ doc("id", "3", "name_s", "nancy", "title_s", "MTS", "dept_s", "Sales"),
+ doc("id", "4", "name_s", "dave", "title_s", "MTS", "dept_s", "Support"),
+ doc("id", "5", "name_s", "tina", "title_s", "VP", "dept_s", "Engineering"));
+ }
+
+ private static SolrInputDocument doc(String... fieldsAndValues) {
+ return new SolrInputDocument(fieldsAndValues);
+ }
+
+ /** Queries the main core and asserts the matches' ids, in index (docid) order. */
+ private static void assertJoin(SolrParams query, String... expectedIds) throws Exception {
+ SolrDocumentList results = solrRule.getSolrClient().query(query).getResults();
+ assertEquals(results.toString(), expectedIds.length, results.getNumFound());
+ assertFieldValues(results, "id", (Object[]) expectedIds);
+ }
+
+ @Test
+ public void testMissingLocalParams() {
+ SolrException e =
+ expectThrows(
+ SolrException.class,
+ () -> solrRule.getSolrClient().query(params("q", "{!auxIndexJoin to=dept_id_s}*:*")));
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(e.getMessage(), e.getMessage().contains("requires"));
+ }
+
+ @Test
+ public void testSameCoreJoin() throws Exception {
+ // nancy and dave (title MTS) each belong to exactly one dept: Sales and Support
+ assertJoin(
+ params("q", "{!auxIndexJoin from=dept_s to=dept_id_s}title_s:MTS", "fl", "id"), "12", "13");
+
+ // from-side subordinate query matches nothing
+ assertJoin(
+ params("q", "{!auxIndexJoin from=dept_s to=dept_id_s}name_s:nosuchperson", "fl", "id"));
+
+ // to-side field has real values, but none equal to any matched from-side value
+ // ("Engineering" is a dept_s value, but no title_s is ever "Engineering")
+ assertJoin(params("q", "{!auxIndexJoin from=dept_s to=title_s}name_s:john", "fl", "id"));
+
+ // a single from-doc resolves to exactly its one department
+ assertJoin(
+ params("q", "{!auxIndexJoin from=dept_s to=dept_id_s}name_s:john", "fl", "id"), "10");
+
+ // variable deref for sub-query parsing, plus defType local param
+ assertJoin(
+ params(
+ "q", "{!auxIndexJoin from=dept_s to=dept_id_s v=$qq}",
+ "qq", "{!dismax qf=name_s}dave",
+ "fl", "id"),
+ "13");
+
+ // fq on the to-side (department) is pushed down alongside the join, mirroring SOLR-3062 for
+ // {!join}: john and tina (title VP) join to Engineering and Marketing, fq narrows to just one
+ assertJoin(
+ params(
+ "q", "{!auxIndexJoin from=dept_s to=dept_id_s}title_s:VP",
+ "fl", "id",
+ "fq", "dept_id_s:Engineering"),
+ "10");
+ }
+
+ @Test
+ public void testCrossCoreJoin() throws Exception {
+ // nancy and dave (title MTS) live in aijoinFromCore; their departments live in this core
+ assertJoin(
+ params(
+ "q",
+ "{!auxIndexJoin from=dept_s to=dept_id_s fromIndex=" + FROM_CORE + "}title_s:MTS",
+ "fl",
+ "id"),
+ "12",
+ "13");
+
+ // fq on the querying (to) core is still pushed down alongside the cross-core join
+ assertJoin(
+ params(
+ "q",
+ "{!auxIndexJoin from=dept_s to=dept_id_s fromIndex=" + FROM_CORE + "}title_s:VP",
+ "fl",
+ "id",
+ "fq",
+ "dept_id_s:Engineering"),
+ "10");
+
+ // the cross-core join must not leak a close on either core
+ CoreContainer coreContainer = solrRule.getCoreContainer();
+ try (SolrCore fromCore = coreContainer.getCore(FROM_CORE);
+ SolrCore toCore = coreContainer.getCore(DEFAULT_TEST_COLLECTION_NAME)) {
+ assertFalse(fromCore.isClosed());
+ assertFalse(toCore.isClosed());
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinBenchmark.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinBenchmark.java
new file mode 100644
index 000000000000..7b97d499ac96
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinBenchmark.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Locale;
+import java.util.concurrent.ExecutorService;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
+import org.apache.lucene.search.join.JoinUtil;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+
+/**
+ * Wall-clock comparison of {@link JoinUtil} (the term-based variant, no global ordinals) against
+ * {@link AuxIndexJoinQuery} on a synthetic M:1 join: ~100K children in a few segments point to ~10K
+ * unique parents in a few segments through single valued sorted string docvalues. A small
+ * child-side term filter selects ~10K children that join to ~1K parents; a second case adds a
+ * parent-side filter on top of the join. Results are not asserted, only timed: each search repeats
+ * {@link #PASSES} times and min/max/avg wall times are printed.
+ *
+ * The whole comparison runs {@link #ROUNDS} times over a growing, mutating index: between rounds
+ * the parent id boundary is raised by 10% and a tenth of the (new) parent population, with all
+ * their children, is rewritten through {@code updateDocument} at random ids below the new boundary
+ * — ids above the old boundary append fresh docs while ids below it clash with existing ones and
+ * replace them, leaving deletes behind. The {@link AuxIndexManager} is opened once and reused by
+ * every round and pass: after each update the first search builds only the missing pair columns,
+ * which is timed separately from the steady-state passes.
+ *
+ *
Plain Java benchmark, run directly with {@code main}, no test framework involved.
+ */
+public class AuxIndexJoinBenchmark {
+
+ private static final String PARENT_ID = "parent_id";
+ private static final String PARENT_ID_FK = "parent_id_FK";
+ private static final String CHILD_ID = "child_id";
+ private static final String TAG = "tag";
+ private static final String COLOR = "color";
+
+ private static final String HOT = "hot";
+ private static final String COLD = "cold";
+ // the parent filter matches one of COLOR_CARDINALITY colors, passing ~1/30 of the joined parents
+ private static final int COLOR_CARDINALITY = 30;
+
+ private static final int NUM_PARENTS = 100_000;
+ private static final int CHILDREN_PER_PARENT = 10;
+ // every HOT_STRIDE'th parent is "hot" and all its children carry the hot tag: the child filter
+ // TermQuery(tag:hot) matches NUM_PARENTS / HOT_STRIDE * CHILDREN_PER_PARENT ~ 10K children
+ // joining to NUM_PARENTS / HOT_STRIDE ~ 1K parents, scattered over all segments
+ private static final int HOT_STRIDE = 10;
+ private static final int PARENT_SEGMENTS = 4;
+ private static final int CHILD_SEGMENTS = 5;
+ private static final int PASSES = 10;
+ // benchmark rounds: between rounds the parent boundary grows by GROWTH_DENOMINATOR'th and
+ // 1/UPDATE_DENOMINATOR of the grown population is rewritten at random ids below the new boundary
+ private static final int ROUNDS = 10;
+ private static final int GROWTH_DENOMINATOR = 10;
+ private static final int UPDATE_DENOMINATOR = 10;
+
+ private long random = System.nanoTime(); // forbiddenApi pushed Random() usage
+
+ private interface SearchTask {
+ TopDocs run() throws IOException;
+ }
+
+ public static void main(String[] args) throws Exception {
+ new AuxIndexJoinBenchmark().runBenchmark();
+ // executor.shutdown();
+ }
+
+ public void runBenchmark() throws Exception {
+ // the auxiliary join index is opened once and reused by every round and pass
+ try (Directory parentsDir = new ByteBuffersDirectory();
+ Directory childrenDir = new ByteBuffersDirectory();
+ Directory joinDir = new ByteBuffersDirectory();
+ ExecutorService executor =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ Runtime.getRuntime().availableProcessors(),
+ new SolrNamedThreadFactory(MethodHandles.lookup().lookupClass().getSimpleName()));
+ AuxIndexManager joinIndex = new AuxIndexManager(joinDir)) {
+ buildIndices(parentsDir, childrenDir);
+ int numParents = NUM_PARENTS;
+ for (int round = 0; round < ROUNDS; round++) {
+ benchmarkRound(round, numParents, parentsDir, childrenDir, joinIndex, executor);
+ if (round < ROUNDS - 1) {
+ numParents += numParents / GROWTH_DENOMINATOR;
+ updateIndices(parentsDir, childrenDir, numParents);
+ }
+ }
+ System.out.println("start warming");
+ try (IndexReader parentsReader = DirectoryReader.open(parentsDir);
+ IndexReader childrenReader = DirectoryReader.open(childrenDir)) {
+ IndexSearcher childrenSearcher = new ParallelIndexSearcher(childrenReader, executor);
+ childrenSearcher.setQueryCache(null);
+ IndexSearcher parentsSearcher = new ParallelIndexSearcher(parentsReader, executor);
+ parentsSearcher.setQueryCache(null);
+
+ Query childFilter = new TermQuery(new Term(TAG, HOT));
+ Query parentFilter = new TermQuery(new Term(COLOR, color(0)));
+
+ for (int i = 0; i < 10000; i++) {
+ exactSearch(
+ parentsSearcher,
+ filterParents(
+ aiJoinChildrenToParents(joinIndex, childFilter, childrenSearcher, executor),
+ parentFilter));
+ if (i % 100 == 0 && i > 0) {
+ System.out.println(i);
+ }
+ }
+ }
+ }
+ }
+
+ private void benchmarkRound(
+ int round,
+ int numParents,
+ Directory parentsDir,
+ Directory childrenDir,
+ AuxIndexManager joinIndex,
+ ExecutorService executor)
+ throws IOException {
+ try (IndexReader parentsReader = DirectoryReader.open(parentsDir);
+ IndexReader childrenReader = DirectoryReader.open(childrenDir)) {
+ System.out.printf(
+ Locale.ROOT,
+ "round %d: parent ids < %d, parents: %d/%d docs (live/max) / %d segments,"
+ + " children: %d/%d docs (live/max) / %d segments%n",
+ round,
+ numParents,
+ parentsReader.numDocs(),
+ parentsReader.maxDoc(),
+ parentsReader.leaves().size(),
+ childrenReader.numDocs(),
+ childrenReader.maxDoc(),
+ childrenReader.leaves().size());
+
+ // plain searchers without caching: AuxIndexJoinQuery is uncacheable anyway, so a cached
+ // JoinUtil query would make the comparison lopsided
+ IndexSearcher childrenSearcher = new ParallelIndexSearcher(childrenReader, executor);
+ childrenSearcher.setQueryCache(null);
+ IndexSearcher parentsSearcher = new ParallelIndexSearcher(parentsReader, executor);
+ parentsSearcher.setQueryCache(null);
+
+ Query childFilter = new TermQuery(new Term(TAG, HOT));
+ Query parentFilter = new TermQuery(new Term(COLOR, color(0)));
+
+ // the first search after a reopen builds the missing pair columns lazily (all of them in
+ // round 0, only the new segments' pairs later), so it's timed apart from the steady-state
+ // passes
+ long buildStart = System.nanoTime();
+ exactSearch(
+ parentsSearcher,
+ aiJoinChildrenToParents(joinIndex, childFilter, childrenSearcher, executor));
+ System.out.printf(
+ Locale.ROOT,
+ "AI join first search (lazy pair build): %.2fms%n",
+ (System.nanoTime() - buildStart) / 1_000_000d);
+
+ // join query creation is inside the timed task on purpose: JoinUtil runs the from-side
+ // selection eagerly in createJoinQuery, AuxIndexJoinQuery lazily in createWeight, so only
+ // create+search is comparable
+ bench(
+ "JoinUtil",
+ () -> exactSearch(parentsSearcher, joinChildrenToParents(childFilter, childrenSearcher)));
+ bench(
+ "AIJoin",
+ () ->
+ exactSearch(
+ parentsSearcher,
+ aiJoinChildrenToParents(joinIndex, childFilter, childrenSearcher, executor)));
+ bench(
+ "JoinUtil + parent filter",
+ () ->
+ exactSearch(
+ parentsSearcher,
+ filterParents(
+ joinChildrenToParents(childFilter, childrenSearcher), parentFilter)));
+ bench(
+ "AIJoin + parent filter",
+ () ->
+ exactSearch(
+ parentsSearcher,
+ filterParents(
+ aiJoinChildrenToParents(joinIndex, childFilter, childrenSearcher, executor),
+ parentFilter)));
+ }
+ }
+
+ private static Query joinChildrenToParents(Query childFilter, IndexSearcher childrenSearcher)
+ throws IOException {
+ // multipleValuesPerDocument=false picks the term-based join, no global ordinals involved
+ return JoinUtil.createJoinQuery(
+ PARENT_ID_FK, false, PARENT_ID, childFilter, childrenSearcher, ScoreMode.None);
+ }
+
+ // private static final ExecutorService executor = Executors.newFixedThreadPool(4);
+
+ private static Query aiJoinChildrenToParents(
+ AuxIndexManager joinIndex,
+ Query childFilter,
+ IndexSearcher childrenSearcher,
+ ExecutorService executor) {
+ return joinIndex.newJoinQuery(PARENT_ID_FK, childFilter, childrenSearcher, PARENT_ID, executor);
+ }
+
+ private static String color(int index) {
+ return "color" + index;
+ }
+
+ private static Query filterParents(Query joinQuery, Query parentFilter) {
+ return new BooleanQuery.Builder()
+ .add(joinQuery, BooleanClause.Occur.MUST)
+ .add(parentFilter, BooleanClause.Occur.FILTER)
+ .build();
+ }
+
+ /**
+ * Runs a top-10 search with an uncapped total hits threshold, so {@code totalHits} is always
+ * exact instead of the default two-arg {@code search(query, 10)}'s early-terminated estimate past
+ * 1000 hits: with that default, JoinUtil and AIJoin visit docs in different orders and cost
+ * estimates and stop at different points, making their reported hit counts incomparable.
+ */
+ private static TopDocs exactSearch(IndexSearcher searcher, Query query) throws IOException {
+ return searcher.search(query, new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE));
+ }
+
+ private static void bench(String name, SearchTask task) throws IOException {
+ long minNs = Long.MAX_VALUE;
+ long maxNs = 0;
+ long totalNs = 0;
+ long hits = -1;
+ for (int pass = 0; pass < PASSES; pass++) {
+ long start = System.nanoTime();
+ TopDocs topDocs = task.run();
+ long elapsed = System.nanoTime() - start;
+ minNs = Math.min(minNs, elapsed);
+ maxNs = Math.max(maxNs, elapsed);
+ totalNs += elapsed;
+ hits = topDocs.totalHits.value();
+ }
+ System.out.printf(
+ Locale.ROOT,
+ "%-26s min=%8.2fms max=%8.2fms avg=%8.2fms hits=%d%n",
+ name,
+ minNs / 1_000_000d,
+ maxNs / 1_000_000d,
+ (double) totalNs / PASSES / 1_000_000d,
+ hits);
+ }
+
+ /**
+ * Writes the two sides with plain {@link IndexWriter}s: every child points to exactly one parent,
+ * {@code parent_id} is unique on the parents side, {@code child_id} is unique on the children
+ * side, both join fields are single valued sorted string docvalues, and periodic commits under
+ * {@link NoMergePolicy} leave each side in a few segments.
+ */
+ private void buildIndices(Directory parentsDir, Directory childrenDir) throws IOException {
+ try (IndexWriter parentsWriter = newBenchWriter(parentsDir);
+ IndexWriter childrenWriter = newBenchWriter(childrenDir)) {
+ int parentsPerSegment = NUM_PARENTS / PARENT_SEGMENTS;
+ int childrenPerSegment = NUM_PARENTS * CHILDREN_PER_PARENT / CHILD_SEGMENTS;
+ int childSeq = 0;
+ for (int p = 0; p < NUM_PARENTS; p++) {
+ parentsWriter.addDocument(parentDoc(p));
+ if ((p + 1) % parentsPerSegment == 0) {
+ parentsWriter.commit();
+ }
+ for (int c = 0; c < CHILDREN_PER_PARENT; c++) {
+ childrenWriter.addDocument(childDoc(p, c));
+ if (++childSeq % childrenPerSegment == 0) {
+ childrenWriter.commit();
+ }
+ }
+ }
+ parentsWriter.commit();
+ childrenWriter.commit();
+ }
+ }
+
+ /**
+ * Simulates incremental growth after the boundary was raised to {@code numParents}: rewrites
+ * {@code numParents / }{@link #UPDATE_DENOMINATOR} parents, each with all its children, at random
+ * ids below the new boundary. Ids above the previous boundary append fresh docs; ids below it
+ * clash with existing {@code parent_id}/{@code child_id} terms and {@code updateDocument}
+ * replaces the old docs, leaving deletes behind in the earlier segments. The single closing
+ * commit adds one new segment per side under {@link NoMergePolicy}.
+ */
+ private void updateIndices(Directory parentsDir, Directory childrenDir, int numParents)
+ throws IOException {
+ try (IndexWriter parentsWriter = newBenchWriter(parentsDir);
+ IndexWriter childrenWriter = newBenchWriter(childrenDir)) {
+ for (int i = 0; i < numParents / UPDATE_DENOMINATOR; i++) {
+ int p = Math.floorMod(random, numParents);
+ random = random * 31L + 31L;
+ parentsWriter.updateDocument(new Term(PARENT_ID, parentId(p)), parentDoc(p));
+ for (int c = 0; c < CHILDREN_PER_PARENT; c++) {
+ childrenWriter.updateDocument(new Term(CHILD_ID, childId(p, c)), childDoc(p, c));
+ }
+ }
+ parentsWriter.commit();
+ childrenWriter.commit();
+ }
+ }
+
+ private IndexWriter newBenchWriter(Directory dir) throws IOException {
+ return new IndexWriter(
+ dir,
+ new IndexWriterConfig()
+ .setMergePolicy(NoMergePolicy.INSTANCE)
+ // pin the flush triggers so the intended segment layout isn't shattered by tiny
+ // default buffers
+ .setMaxBufferedDocs(IndexWriter.MAX_DOCS)
+ .setRAMBufferSizeMB(256));
+ }
+
+ private static String parentId(int p) {
+ return "parent" + p;
+ }
+
+ private static String childId(int p, int c) {
+ return "child" + p + "." + c;
+ }
+
+ private static Document parentDoc(int p) {
+ String parentId = parentId(p);
+ Document doc = new Document();
+ doc.add(new StringField(PARENT_ID, parentId, Field.Store.NO));
+ doc.add(new SortedDocValuesField(PARENT_ID, new BytesRef(parentId)));
+ // cycle colors by hot rank, not by p: a plain p % COLOR_CARDINALITY would correlate with
+ // the p % HOT_STRIDE hot selection and the filter wouldn't thin the joined parents
+ doc.add(new StringField(COLOR, color((p / HOT_STRIDE) % COLOR_CARDINALITY), Field.Store.NO));
+ return doc;
+ }
+
+ private static Document childDoc(int p, int c) {
+ Document doc = new Document();
+ doc.add(new StringField(CHILD_ID, childId(p, c), Field.Store.NO));
+ doc.add(new SortedDocValuesField(PARENT_ID_FK, new BytesRef(parentId(p))));
+ doc.add(new StringField(TAG, p % HOT_STRIDE == 0 ? HOT : COLD, Field.Store.NO));
+ return doc;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/ParallelIndexSearcher.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/ParallelIndexSearcher.java
new file mode 100644
index 000000000000..eb0541554d71
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/ParallelIndexSearcher.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.util.List;
+import java.util.concurrent.Executor;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+
+final class ParallelIndexSearcher extends IndexSearcher {
+ ParallelIndexSearcher(IndexReader r, Executor executor) {
+ super(r, executor);
+ }
+
+ @Override
+ protected LeafSlice[] slices(List leaves) {
+ return slices(leaves, 1000, 5, false);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoin.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoin.java
new file mode 100644
index 000000000000..a4798a289c09
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoin.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.UnaryOperator;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LRUQueryCache;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.join.JoinUtil;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.jspecify.annotations.NonNull;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Joins a children index to a parents index with {@link JoinUtil}. Children reference parents via a
+ * single valued {@link SortedSetDocValuesField} {@code parent_id_FK} pointing to the parents'
+ * single valued {@code parent_id} (M:1).
+ */
+public class TestAuxIndexJoin extends SolrTestCase {
+
+ private static final String ID = "id";
+ private static final String PARENT_ID = "parent_id";
+ private static final String PARENT_ID_FK = "parent_id_FK";
+ private static final String COLOR = "color";
+
+ private static final String[] COLORS = {"red", "green", "blue"};
+ private static final int CHILDREN_PER_PARENT = 5;
+
+ private static ExecutorService executor;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @BeforeClass
+ public static void beforeClass() {
+ executor =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ random().nextInt(1, 3),
+ new SolrNamedThreadFactory(MethodHandles.lookup().lookupClass().getSimpleName()));
+ }
+
+ @AfterClass
+ public static void afterClass() throws InterruptedException {
+ executor.shutdown();
+ assertTrue(
+ "suite-wide executor did not shut down in time",
+ executor.awaitTermination(60, TimeUnit.SECONDS));
+ }
+
+ /** Shared per-test auxiliary join index: pair columns are built lazily by the first search. */
+ private AuxIndexManager joinIndex;
+
+ private Directory joinDir;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ joinDir = newDirectory();
+ // none of these affect join correctness, only internal segment layout, refresh latency, and
+ // dead-pair reap timing -- randomized to exercise all combinations
+ AuxIndexJoinConfig config =
+ new AuxIndexJoinConfig()
+ .setSingleFieldPerSegment(random().nextBoolean())
+ .setBlockingRefresh(random().nextBoolean())
+ .setUseFromSideThreads(random().nextBoolean())
+ .setSweepSamplingInterval(TestUtil.nextInt(random(), -1, 2), TimeUnit.MINUTES);
+ joinIndex = new AuxIndexManager(joinDir, config);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(joinIndex, joinDir);
+ super.tearDown();
+ }
+
+ /** Two indices: parents and their children, segmented by intermediate commits. */
+ private static class ParentChildIndices implements Closeable {
+ final Directory parentsDir = newDirectory();
+ final Directory childrenDir = newDirectory();
+ final RandomIndexWriter parentsWriter;
+ final RandomIndexWriter childrenWriter;
+ // childId -> parentId, insertion order
+ final Map parentIdByChildId = new TreeMap<>();
+ final Map colorByParentId = new TreeMap<>();
+
+ ParentChildIndices() throws IOException {
+ // NoMergePolicy keeps the segments created by the intermediate commits below
+ parentsWriter =
+ new RandomIndexWriter(
+ random(),
+ parentsDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ childrenWriter =
+ new RandomIndexWriter(
+ random(),
+ childrenDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ int numParents = atLeast(10);
+ int childSeq = 0;
+ for (int p = 0; p < numParents; p++) {
+ String parentId = "parent" + p;
+ String color = RandomPicks.randomFrom(random(), COLORS);
+ colorByParentId.put(parentId, color);
+ parentsWriter.addDocument(parentDoc(parentId, color));
+ for (int c = 0; c < CHILDREN_PER_PARENT; c++) {
+ String childId = "child" + childSeq++;
+ parentIdByChildId.put(childId, parentId);
+ childrenWriter.addDocument(childDoc(childId, parentId));
+ }
+ if (p % 3 == 2) {
+ parentsWriter.commit();
+ childrenWriter.commit();
+ }
+ }
+ }
+
+ List childrenOf(String parentId) {
+ List children = new ArrayList<>();
+ parentIdByChildId.forEach(
+ (childId, pid) -> {
+ if (pid.equals(parentId)) {
+ children.add(childId);
+ }
+ });
+ return children;
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(parentsWriter, childrenWriter, parentsDir, childrenDir);
+ }
+ }
+
+ private static Document parentDoc(String parentId, String color) {
+ Document doc = new Document();
+ doc.add(new StringField(PARENT_ID, parentId, Field.Store.YES));
+ doc.add(new SortedSetDocValuesField(PARENT_ID, new BytesRef(parentId)));
+ doc.add(new StringField(COLOR, color, Field.Store.YES));
+ return doc;
+ }
+
+ private static Document childDoc(String childId, String parentId) {
+ Document doc = new Document();
+ doc.add(new StringField(ID, childId, Field.Store.YES));
+ doc.add(new StringField(PARENT_ID_FK, parentId, Field.Store.YES));
+ doc.add(new SortedSetDocValuesField(PARENT_ID_FK, new BytesRef(parentId)));
+ return doc;
+ }
+
+ private static Query joinChildrenToParents(Query fromQuery, IndexSearcher childrenSearcher)
+ throws IOException {
+ // any score mode matches the same parents
+ ScoreMode scoreMode = RandomPicks.randomFrom(random(), ScoreMode.values());
+ return JoinUtil.createJoinQuery(
+ PARENT_ID_FK, true, PARENT_ID, fromQuery, childrenSearcher, scoreMode);
+ }
+
+ private static Set searchParentIds(IndexSearcher parentsSearcher, Query query)
+ throws IOException {
+ TopDocs topDocs = parentsSearcher.search(query, parentsSearcher.getIndexReader().maxDoc());
+ Set parentIds = new TreeSet<>();
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ parentIds.add(parentsSearcher.storedFields().document(scoreDoc.doc).get(PARENT_ID));
+ }
+ return parentIds;
+ }
+
+ /**
+ * Searches the same join through {@link JoinUtil} and through the auxiliary join index and
+ * asserts both agree; {@code joinDecorator} wraps each join query the same way before searching.
+ */
+ private Set searchParentIdsBothJoins(
+ IndexSearcher parentsSearcher,
+ Query fromQuery,
+ IndexSearcher childrenSearcher,
+ UnaryOperator joinDecorator)
+ throws IOException {
+ Query joinUtilQuery = joinChildrenToParents(fromQuery, childrenSearcher);
+ Set joinUtilParents =
+ searchParentIds(parentsSearcher, joinDecorator.apply(joinUtilQuery));
+ Query aiJoinQuery = createAiJoinQuery(joinIndex, fromQuery, childrenSearcher);
+ assertEquals(
+ "AuxIndexJoinQuery disagrees with JoinUtil",
+ joinUtilParents,
+ searchParentIds(parentsSearcher, joinDecorator.apply(aiJoinQuery)));
+ return joinUtilParents;
+ }
+
+ private @NonNull Query createAiJoinQuery(
+ AuxIndexManager joinIndexParam, Query fromQuery, IndexSearcher childrenSearcher) {
+ return joinIndexParam.newJoinQuery(
+ PARENT_ID_FK,
+ fromQuery,
+ random().nextBoolean() ? cachedSearcher(childrenSearcher) : childrenSearcher,
+ PARENT_ID,
+ rarely() ? null : executor);
+ }
+
+ private Set searchParentIdsBothJoins(
+ IndexSearcher parentsSearcher, Query fromQuery, IndexSearcher childrenSearcher)
+ throws IOException {
+ return searchParentIdsBothJoins(
+ parentsSearcher, fromQuery, childrenSearcher, UnaryOperator.identity());
+ }
+
+ private static Query anyOfChildren(Set childIds) {
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ for (String childId : childIds) {
+ builder.add(new TermQuery(new Term(ID, childId)), BooleanClause.Occur.SHOULD);
+ }
+ return builder.build();
+ }
+
+ private static Set randomChildrenSubset(ParentChildIndices indices) {
+ Set selected = new TreeSet<>();
+ for (String childId : indices.parentIdByChildId.keySet()) {
+ if (random().nextBoolean()) {
+ selected.add(childId);
+ }
+ }
+ if (selected.isEmpty()) {
+ selected.add(RandomPicks.randomFrom(random(), indices.parentIdByChildId.keySet()));
+ }
+ return selected;
+ }
+
+ private static Set expectedParents(ParentChildIndices indices, Set childIds) {
+ Set expected = new TreeSet<>();
+ for (String childId : childIds) {
+ expected.add(indices.parentIdByChildId.get(childId));
+ }
+ return expected;
+ }
+
+ public void testAIJoinRandomChildrenSubset() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ assertTrue("children index should be segmented", childrenReader.leaves().size() > 1);
+ assertTrue("parents index should be segmented", parentsReader.leaves().size() > 1);
+
+ Set selectedChildren = randomChildrenSubset(indices);
+ // the first search builds the missing pair columns into the join index on demand
+ assertEquals(
+ expectedParents(indices, selectedChildren),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ anyOfChildren(selectedChildren),
+ newSearcher(childrenReader)));
+ }
+
+ // grow both sides and reopen: the next search finds the old pair columns persisted and
+ // lazily builds only the pairs involving the new segments
+ String newParentId = "parentNew";
+ String newParentColor = RandomPicks.randomFrom(random(), COLORS);
+ indices.colorByParentId.put(newParentId, newParentColor);
+ indices.parentsWriter.addDocument(parentDoc(newParentId, newParentColor));
+ String newChildId = "childNew";
+ String oldChildId = RandomPicks.randomFrom(random(), indices.parentIdByChildId.keySet());
+ indices.parentIdByChildId.put(newChildId, newParentId);
+ indices.childrenWriter.addDocument(childDoc(newChildId, newParentId));
+ indices.parentsWriter.commit();
+ indices.childrenWriter.commit();
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ assertEquals(
+ Set.of(newParentId),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ new TermQuery(new Term(ID, newChildId)),
+ newSearcher(childrenReader)));
+
+ // one query spanning old and new children joins to their old and new parents
+ assertEquals(
+ Set.of(indices.parentIdByChildId.get(oldChildId), newParentId),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ anyOfChildren(new TreeSet<>(List.of(oldChildId, newChildId))),
+ newSearcher(childrenReader)));
+ }
+ }
+ }
+
+ /**
+ * A from-side segment whose join field holds a single distinct value (one parent referenced by
+ * every child) must still answer lookups for to-side terms absent from it. The per-segment term
+ * dictionary then has {@code getValueCount() == 1}, which used to seed the lookup hash at size 1
+ * -- a table {@code add()} fills to 100%, so probing for the orphan parent's term never found an
+ * empty slot and spun forever.
+ */
+ public void testFromSegmentWithSingleDistinctValue() throws Exception {
+ try (Directory childrenDir = newDirectory();
+ Directory parentsDir = newDirectory()) {
+ // a plain IndexWriter (not RandomIndexWriter) keeps all children in one in-RAM segment until
+ // close, so the from-segment's term dictionary has exactly one distinct value
+ try (IndexWriter childrenWriter =
+ new IndexWriter(
+ childrenDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int c = 0; c < 5; c++) {
+ childrenWriter.addDocument(childDoc("child" + c, "onlyParent"));
+ }
+ }
+ try (IndexWriter parentsWriter =
+ new IndexWriter(
+ parentsDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ // one parent referenced by every child, plus an orphan parent whose term is absent from the
+ // children's single-value term dictionary -- the lookup that used to hang
+ parentsWriter.addDocument(parentDoc("onlyParent", "red"));
+ parentsWriter.addDocument(parentDoc("orphanParent", "blue"));
+ }
+ try (IndexReader childrenReader = DirectoryReader.open(childrenDir);
+ IndexReader parentsReader = DirectoryReader.open(parentsDir)) {
+ assertEquals(
+ Set.of("onlyParent"),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ anyOfChildren(new TreeSet<>(List.of("child0", "child1"))),
+ newSearcher(childrenReader)));
+ }
+ }
+ }
+
+ public void testUpdateChildParentIdFK() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ // repeated re-pointing exercises the sidecar join index's update path many times over,
+ // not just a single one-off update
+ for (int iter = 0; iter < atLeast(5); iter++) {
+ String childId = RandomPicks.randomFrom(random(), indices.parentIdByChildId.keySet());
+ String oldParentId = indices.parentIdByChildId.get(childId);
+ String newParentId;
+ do {
+ newParentId = RandomPicks.randomFrom(random(), indices.colorByParentId.keySet());
+ } while (newParentId.equals(oldParentId));
+
+ // SortedSetDocValues can't be updated in place, so replace the whole child document
+ indices.childrenWriter.updateDocument(
+ new Term(ID, childId), childDoc(childId, newParentId));
+ indices.parentIdByChildId.put(childId, newParentId);
+
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ IndexSearcher childrenSearcher = newSearcher(childrenReader);
+ IndexSearcher parentsSearcher = newSearcher(parentsReader);
+
+ assertEquals(
+ Set.of(newParentId),
+ searchParentIdsBothJoins(
+ parentsSearcher, new TermQuery(new Term(ID, childId)), childrenSearcher));
+
+ // the old parent is reachable only while it still has other children pointing at it
+ Set expectedOldParent =
+ indices.childrenOf(oldParentId).isEmpty() ? Set.of() : Set.of(oldParentId);
+ assertEquals(
+ expectedOldParent,
+ searchParentIdsBothJoins(
+ parentsSearcher,
+ new TermQuery(new Term(PARENT_ID_FK, oldParentId)),
+ childrenSearcher));
+ }
+ }
+ }
+ }
+
+ public void testUpdateParentId() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ // repeated renaming exercises the sidecar join index's update path many times over, not
+ // just a single one-off rename
+ for (int iter = 0; iter < atLeast(5); iter++) {
+ String parentId = RandomPicks.randomFrom(random(), indices.colorByParentId.keySet());
+ String renamedParentId = parentId + "-renamed" + iter;
+ String color = indices.colorByParentId.remove(parentId);
+ indices.parentsWriter.updateDocument(
+ new Term(PARENT_ID, parentId), parentDoc(renamedParentId, color));
+ indices.colorByParentId.put(renamedParentId, color);
+
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ // children still point at the old id, so they join to nothing
+ assertEquals(
+ Set.of(),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ new TermQuery(new Term(PARENT_ID_FK, parentId)),
+ newSearcher(childrenReader)));
+ }
+
+ // re-point one child at the renamed parent and the join works again, if any of this
+ // parent's children haven't already been re-pointed elsewhere by an earlier iteration
+ List children = indices.childrenOf(parentId);
+ if (children.isEmpty()) {
+ continue;
+ }
+ String childId = RandomPicks.randomFrom(random(), children);
+ indices.childrenWriter.updateDocument(
+ new Term(ID, childId), childDoc(childId, renamedParentId));
+ indices.parentIdByChildId.put(childId, renamedParentId);
+
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ assertEquals(
+ Set.of(renamedParentId),
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ new TermQuery(new Term(ID, childId)),
+ newSearcher(childrenReader)));
+ }
+ }
+ }
+ }
+
+ public void testAIJoinWithParentTermFilter() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader();
+ Directory joinDir = newDirectory();
+ AuxIndexManager joinIndex = new AuxIndexManager(joinDir)) {
+ Set selectedChildren = randomChildrenSubset(indices);
+ String color = RandomPicks.randomFrom(random(), COLORS);
+
+ Query aiJoinQuery =
+ createAiJoinQuery(
+ joinIndex, anyOfChildren(selectedChildren), newSearcher(childrenReader));
+
+ Query filteredJoin =
+ new BooleanQuery.Builder()
+ .add(aiJoinQuery, BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term(COLOR, color)), BooleanClause.Occur.FILTER)
+ .build();
+
+ Set expected = new TreeSet<>();
+ for (String parentId : expectedParents(indices, selectedChildren)) {
+ if (color.equals(indices.colorByParentId.get(parentId))) {
+ expected.add(parentId);
+ }
+ }
+ assertEquals(expected, searchParentIds(newSearcher(parentsReader), filteredJoin));
+ }
+ }
+ }
+
+ /**
+ * Not a correctness test but a tracing scenario for the repeated-ForeignKeyColumn-load issue;
+ * asserts nothing, read the {@code AIJOIN} log lines instead.
+ *
+ * The FILTER clause is added before the join clause and filters on {@code
+ * parent_id=parent0}, a term present in only the first parents segment. Per to-leaf, {@code
+ * BooleanWeight.scorerSupplier} asks the clauses in order and bails out with {@code null} on the
+ * first required clause that has no matches -- so for every to-segment except the first one the
+ * join's scorerSupplier is never invoked, no {@code JoinIndexScorerSupplier} is created, and the
+ * pairs (every from-segment x that to-segment) are never written to the sidecar.
+ *
+ *
Expected log shape: pass 1 loads all FK columns and builds only the first to-segment's
+ * pairs; passes 2..3 show {@code evt=weight} with the same {@code pairsMissing} again, {@code
+ * evt=fkload} repeating for every from-segment, and no {@code evt=build} -- the profiler-visible
+ * waste. The unfiltered pass then visits every to-segment and writes the remaining pairs, after
+ * which the final filtered pass runs with {@code pairsMissing=0} and no {@code evt=fkload}.
+ */
+ @Ignore
+ public void testTraceFkReloadForSkippedToSegments() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ IndexSearcher parentsSearcher = newSearcher(parentsReader);
+ Query aiJoinQuery =
+ createAiJoinQuery(
+ joinIndex,
+ anyOfChildren(randomChildrenSubset(indices)),
+ newSearcher(childrenReader));
+ Query filteredJoin =
+ new BooleanQuery.Builder()
+ // FILTER first: a to-segment without the term short-circuits BooleanWeight
+ // before the join clause's scorerSupplier is consulted
+ .add(new TermQuery(new Term(PARENT_ID, "parent0")), BooleanClause.Occur.FILTER)
+ .add(aiJoinQuery, BooleanClause.Occur.MUST)
+ .build();
+ for (int pass = 1; pass <= 3; pass++) {
+ log.info(
+ "AIJOIN-TRACE pass={} filtered join; beyond pass 1 expect repeated pairsMissing"
+ + " and evt=fkload with no evt=build",
+ pass);
+ searchParentIds(parentsSearcher, filteredJoin);
+ }
+ log.info(
+ "AIJOIN-TRACE unfiltered join: every to-segment is visited, remaining pairs written");
+ searchParentIds(parentsSearcher, aiJoinQuery);
+ log.info("AIJOIN-TRACE final filtered pass: expect pairsMissing=0 and no evt=fkload");
+ searchParentIds(parentsSearcher, filteredJoin);
+ }
+ }
+ }
+
+ public void testJoinWithParentTermFilter() throws Exception {
+ try (ParentChildIndices indices = new ParentChildIndices()) {
+ try (IndexReader childrenReader = indices.childrenWriter.getReader();
+ IndexReader parentsReader = indices.parentsWriter.getReader()) {
+ Set selectedChildren = randomChildrenSubset(indices);
+ String color = RandomPicks.randomFrom(random(), COLORS);
+
+ Set expected = new TreeSet<>();
+ for (String parentId : expectedParents(indices, selectedChildren)) {
+ if (color.equals(indices.colorByParentId.get(parentId))) {
+ expected.add(parentId);
+ }
+ }
+ assertEquals(
+ expected,
+ searchParentIdsBothJoins(
+ newSearcher(parentsReader),
+ anyOfChildren(selectedChildren),
+ newSearcher(childrenReader),
+ join ->
+ new BooleanQuery.Builder()
+ .add(join, BooleanClause.Occur.MUST)
+ .add(new TermQuery(new Term(COLOR, color)), BooleanClause.Occur.FILTER)
+ .build()));
+ }
+ }
+ }
+
+ // presumably keep it in AuxIndexManager
+ private static IndexSearcher cachedSearcher(IndexSearcher fromSearcher) {
+ IndexSearcher cachedFromSearcher =
+ new IndexSearcher(
+ fromSearcher.getIndexReader() // , executor // pointless
+ );
+ cachedFromSearcher.setQueryCache(
+ new LRUQueryCache(
+ fromSearcher.getLeafContexts().size() + 1,
+ fromSearcher.getIndexReader().maxDoc() / 8 * 2));
+ cachedFromSearcher.setQueryCachingPolicy(
+ new QueryCachingPolicy() {
+ @Override
+ public boolean shouldCache(Query query) {
+ return true;
+ }
+
+ @Override
+ public void onUse(Query query) {}
+ });
+ return cachedFromSearcher;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoinMergePolicy.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoinMergePolicy.java
new file mode 100644
index 000000000000..9b636ab04a29
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestAuxIndexJoinMergePolicy.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.solr.SolrTestCase;
+
+/**
+ * Real (non-mocked) integration smoke test for {@link AuxIndexJoinMergePolicy}: builds a segmented
+ * children/parents pair, forces {@link AuxIndexJoinQuery} to populate the sidecar join index, then
+ * mass deletes and force-merges both sides so the sidecar's previously-built pair columns go stale,
+ * and checks the policy actually notices and reaps them -- not just that it runs without throwing.
+ */
+public class TestAuxIndexJoinMergePolicy extends SolrTestCase {
+
+ private static final String ID = "id";
+ private static final String PARENT_ID = "parent_id";
+ private static final String PARENT_ID_FK = "parent_id_FK";
+
+ private Directory parentsDir;
+ private Directory childrenDir;
+ private Directory joinDir;
+ private RandomIndexWriter parentsWriter;
+ private RandomIndexWriter childrenWriter;
+ private AuxIndexManager joinIndex;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ parentsDir = newDirectory();
+ childrenDir = newDirectory();
+ // NoMergePolicy keeps every intermediate commit as its own segment, so the join index has
+ // several distinct (from-segment, to-segment) pairs to build, not just one
+ parentsWriter =
+ new RandomIndexWriter(
+ random(),
+ parentsDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ childrenWriter =
+ new RandomIndexWriter(
+ random(),
+ childrenDir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ joinDir = newDirectory();
+ joinIndex = new AuxIndexManager(joinDir);
+ // this test drives many onCreateWeight calls back to back, well inside the default one-minute
+ // sampling interval, and asserts on the reaper noticing every one of them
+ joinIndex.mergePolicy.setSweepInterval(0, TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(joinIndex, joinDir, parentsWriter, childrenWriter, parentsDir, childrenDir);
+ super.tearDown();
+ }
+
+ private static Document parentDoc(String parentId) {
+ Document doc = new Document();
+ doc.add(new StringField(PARENT_ID, parentId, Field.Store.YES));
+ doc.add(new SortedSetDocValuesField(PARENT_ID, new BytesRef(parentId)));
+ return doc;
+ }
+
+ private static Document childDoc(String childId, String parentId) {
+ Document doc = new Document();
+ doc.add(new StringField(ID, childId, Field.Store.YES));
+ doc.add(new StringField(PARENT_ID_FK, parentId, Field.Store.YES));
+ doc.add(new SortedSetDocValuesField(PARENT_ID_FK, new BytesRef(parentId)));
+ return doc;
+ }
+
+ /**
+ * Adds {@code numParents} parents (3 children each), committing every 3 parents so both sides end
+ * up segmented rather than a single flush. Returns the added parent ids.
+ */
+ private List addParentsAndChildren(String parentIdPrefix, int numParents)
+ throws IOException {
+ List parentIds = new ArrayList<>();
+ for (int p = 0; p < numParents; p++) {
+ String parentId = parentIdPrefix + p;
+ parentIds.add(parentId);
+ parentsWriter.addDocument(parentDoc(parentId));
+ for (int c = 0; c < 3; c++) {
+ childrenWriter.addDocument(childDoc(parentId + "_child" + c, parentId));
+ }
+ if (p % 3 == 2) {
+ parentsWriter.commit();
+ childrenWriter.commit();
+ }
+ }
+ parentsWriter.commit();
+ childrenWriter.commit();
+ return parentIds;
+ }
+
+ /**
+ * Deletes every parent in {@code parentIds} (and its children), then force-merges both sides down
+ * to a single segment each -- changing both sides' segment identities so every sidecar pair
+ * column referencing them goes stale.
+ */
+ private void deleteParentsAndForceMerge(List parentIds) throws IOException {
+ for (String parentId : parentIds) {
+ parentsWriter.deleteDocuments(new Term(PARENT_ID, parentId));
+ for (int c = 0; c < 3; c++) {
+ childrenWriter.deleteDocuments(new Term(ID, parentId + "_child" + c));
+ }
+ }
+ // NoMergePolicy (set at construction, to guarantee segmentation above) blocks forceMerge too,
+ // so swap in a real policy just for these force merges
+ parentsWriter.w.getConfig().setMergePolicy(new TieredMergePolicy());
+ childrenWriter.w.getConfig().setMergePolicy(new TieredMergePolicy());
+ parentsWriter.forceMerge(1);
+ childrenWriter.forceMerge(1);
+ parentsWriter.commit();
+ childrenWriter.commit();
+ }
+
+ /**
+ * Runs the join query for every child against every parent and returns the matched parent ids,
+ * forcing {@link AuxIndexJoinQuery} to fully execute (not just build its {@link
+ * org.apache.lucene.search.Weight}) so missing pair columns actually get built into the sidecar.
+ */
+ private Set searchAllParents(
+ IndexSearcher parentsSearcher, IndexSearcher childrenSearcher) throws IOException {
+ Query aiJoinQuery =
+ joinIndex.newJoinQuery(PARENT_ID_FK, new MatchAllDocsQuery(), childrenSearcher, PARENT_ID);
+ TopDocs topDocs =
+ parentsSearcher.search(aiJoinQuery, parentsSearcher.getIndexReader().maxDoc());
+ Set parentIds = new TreeSet<>();
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ parentIds.add(parentsSearcher.storedFields().document(scoreDoc.doc).get(PARENT_ID));
+ }
+ return parentIds;
+ }
+
+ public void testDeadPairsAreReapedAfterForceMerge() throws Exception {
+ List firstBatch = addParentsAndChildren("gen1-", atLeast(15));
+
+ try (IndexReader childrenReader = childrenWriter.getReader();
+ IndexReader parentsReader = parentsWriter.getReader()) {
+ assertTrue(
+ "children index should be segmented before merging", childrenReader.leaves().size() > 1);
+ assertTrue(
+ "parents index should be segmented before merging", parentsReader.leaves().size() > 1);
+
+ // populates the sidecar with every (child-segment, parent-segment) pair currently needed
+ assertEquals(
+ new TreeSet<>(firstBatch),
+ searchAllParents(newSearcher(parentsReader), newSearcher(childrenReader)));
+ }
+
+ assertEquals(
+ "nothing should look dead yet, nothing has disappeared",
+ 0,
+ joinIndex.mergePolicy.pendingPairRemovalsCount());
+ assertEquals(0, joinIndex.mergePolicy.droppedSegmentCount());
+
+ // mass removal: drop every other parent (and its children), then force-merge both sides --
+ // every pair column built above referenced a from/to segment that no longer exists
+ List deadFromFirstBatch = new ArrayList<>();
+ List survivingParents = new ArrayList<>();
+ for (int i = 0; i < firstBatch.size(); i++) {
+ (i % 2 == 0 ? deadFromFirstBatch : survivingParents).add(firstBatch.get(i));
+ }
+ deleteParentsAndForceMerge(deadFromFirstBatch);
+
+ try (IndexReader childrenReader = childrenWriter.getReader();
+ IndexReader parentsReader = parentsWriter.getReader()) {
+ assertEquals(1, childrenReader.leaves().size());
+ assertEquals(1, parentsReader.leaves().size());
+
+ // re-running the same join rebuilds pairs against the new segments and, in the process,
+ // notices the old pairs vanished
+ assertEquals(
+ new TreeSet<>(survivingParents),
+ searchAllParents(newSearcher(parentsReader), newSearcher(childrenReader)));
+ }
+
+ joinIndex.waitForMerges();
+
+ assertTrue(
+ "the old pair columns should have been recognized as dead",
+ joinIndex.mergePolicy.pendingPairRemovalsCount() > 0);
+ assertTrue(
+ "AuxIndexJoinMergePolicy should have reaped at least the one dead sidecar segment",
+ joinIndex.mergePolicy.droppedSegmentCount() >= 1);
+
+ // do it again: grow, mass-delete, force-merge -- the reaper should keep working, not just
+ // fire once. Force-merging an index that already sits at a single, fully-live segment is a
+ // no-op (nothing to reclaim), so the surviving segment above keeps its identity and its
+ // sidecar pair never goes stale on its own; adding one more small batch afterwards guarantees
+ // a genuinely new pair needs building, which is what actually triggers the sidecar's next
+ // commit (and so the next reap opportunity) -- the reaper is piggybacked on writes, not on a
+ // background timer.
+ int droppedSoFar = joinIndex.mergePolicy.droppedSegmentCount();
+ List secondBatch = addParentsAndChildren("gen2-", atLeast(15));
+ try (IndexReader childrenReader = childrenWriter.getReader();
+ IndexReader parentsReader = parentsWriter.getReader()) {
+ Set expected = new TreeSet<>(survivingParents);
+ expected.addAll(secondBatch);
+ assertEquals(
+ expected, searchAllParents(newSearcher(parentsReader), newSearcher(childrenReader)));
+ }
+
+ deleteParentsAndForceMerge(secondBatch);
+ List thirdBatch = addParentsAndChildren("gen3-", 1);
+
+ try (IndexReader childrenReader = childrenWriter.getReader();
+ IndexReader parentsReader = parentsWriter.getReader()) {
+ Set expected = new TreeSet<>(survivingParents);
+ expected.addAll(thirdBatch);
+ assertEquals(
+ expected, searchAllParents(newSearcher(parentsReader), newSearcher(childrenReader)));
+ }
+
+ joinIndex.waitForMerges();
+
+ assertTrue(
+ "a second round of mass deletes + force merge should reap more dead segments",
+ joinIndex.mergePolicy.droppedSegmentCount() > droppedSoFar);
+ }
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/auxindexjoin-query-parser.adoc b/solr/solr-ref-guide/modules/query-guide/pages/auxindexjoin-query-parser.adoc
new file mode 100644
index 000000000000..a862f8eaf4c5
--- /dev/null
+++ b/solr/solr-ref-guide/modules/query-guide/pages/auxindexjoin-query-parser.adoc
@@ -0,0 +1,142 @@
+= Auxiliary Index Join Query Parser
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License
+
+`auxIndexJoin` (which stands for Auxiliary Index Join) Query Parser is similar to xref:join-query-parser.adoc[], but uses a lazily written sidecar index for faster joins.
+
+Instead of using Lucene's join utilities, the parser matches documents through a dedicated join index that is maintained alongside the core's main index.
+The join index is populated lazily: if the inner (`from`-side) query doesn't hit a certain segment, the join index isn't written for it; and if `{!auxIndexJoin}` is intersected (`AND`/`+`) with a query that doesn't hit a certain segment on the outer (`to`) side, the corresponding join index columns aren't written either.
+
+Like the Join query parser, Solr runs a subquery (the `v` parameter), gathers the values that matching documents have in a `from` field, and returns documents where those values are contained in a `to` field.
+
+For example:
+
+[source,text]
+----
+q={!auxIndexJoin from=manu_id_s to=id}title:ipod
+----
+
+== Parameters
+
+This query parser takes the following parameters:
+
+`from`::
++
+[%autowidth,frame=none]
+|===
+s|Required |Default: none
+|===
++
+The "foreign key" field name, collected while enumerating the subordinate query.
+So far, it should be single value string field with docValues enabled.
+
+`to`::
++
+[%autowidth,frame=none]
+|===
+s|Required |Default: none
+|===
++
+The "primary key" field name looked up in the local core's index.
+So far, it should be single value string field with docValues enabled.
+
+`fromIndex`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: processing core
+|===
++
+The name of the core to run the "from" query (`v` parameter) on and where "from" values are gathered.
+If this parameter is not defined, it defaults to the processing core.
+Cross-core joins are the primary use case for `auxIndexJoin`, so this is useful when the "from" values live in a different core on the same node.
+
+== Configuration
+
+`auxIndexJoin` query parser *must be* registered in `solrconfig.xml` as a ``.
+A sidecar join index is opened per core when the core loads, in a directory under the core's `dataDir`, and closed when the core closes.
+
+`dir`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `aux-index-join`
+|===
++
+The init parameter for the directory holding the sidecar join index.
+Resolved relative to the core's `dataDir` unless an absolute path is given.
+
+`singleFieldPerSegment`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `false`
+|===
++
+If `true`, each pair column is flushed into its own sidecar segment.
+Otherwise, every pair column built in the same round is batched into a single segment (the default), trading a longer sweep for less storage overhead.
+
+`blockingRefresh`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `true`
+|===
++
+If `true`, writing a batch of pair columns blocks until the sidecar's searcher manager is refreshed past it, so the freshly built pairs are immediately visible to the caller that triggered the build.
+
+`useFromSideThreads`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `true`
+|===
++
+If `true` (the default), searching the from-side segments and loading foreigh key columns is parallelized across the executor threads described in <>.
+Note it's actual only if <> is enabled and treated as `false` otherwise.
+If `false`, the from-side segments are loaded sequentially on the calling thread, bounding the build to a single thread.
+
+`sweepSamplingInterval`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `60`
+|===
++
+How often (in seconds) the dead-pair reaper samples searcher state while the sidecar index is being read.
+Calls arriving sooner than this interval after the last accepted sample are skipped.
+Pass a non-positive value to sample on every call.
+
+[source,xml]
+----
+
+ aijoin
+ false
+ true
+ true
+ 60
+
+----
+
+[[segment-level-parallelism]]
+== Segment-level Parallelism
+
+The algorithm utilizes multiple threads on _both_ sides if multiple threads are available. So, set xref:configuration-guide:configuring-solr-xml.adoc#indexSearcherExecutorThreads[indexSearcherExecutorThreads] to `-1` or `>0` in the `solr.xml` file. Note the emphasis on _both_ above: using the xref:common-query-parameters.adoc#multithreaded-parameter[`multiThreaded`] request parameter confines the inner (`from`-side) query to a single thread, limiting performance.
+
+== Disclaimer
+
+`auxIndexJoin` is completely experimental, which is why it requires explicit configuration. So, far it's evalueated within single relation (from-to index pair) only, but it should support many ones in principle. It's parameters/usage is more subject to change with less backwards compatibility concern.
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
index 7acd8024fee5..7c8f1765246a 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
@@ -454,7 +454,9 @@ Similar to using <>, when ear
|===
This parameter set to `true` or `false` controls if Solr may use more than one thread to satisfy the request.
-A `true` value presently allows the IndexSearcher to search across Lucene's segments in parallel, and the xref:configuration-guide:configuring-solr-xml.adoc#indexSearcherExecutorThreads[indexSearcherExecutorThreads] value can be customised in the `solr.xml` file.
+A `true` value presently allows the IndexSearcher to search across Lucene's segments in parallel,
+and the xref:configuration-guide:configuring-solr-xml.adoc#indexSearcherExecutorThreads[indexSearcherExecutorThreads]
+value can be customized in the `solr.xml` file.
This parameter is ignored in the presence of `&segmentsTerminateEarly=true` (future work may enable it).
This is a new parameter and is considered experimental and subject to change or removal in subsequent releases.
Please share your feedback and experiences with it on our mailing lists.
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc
index 7a8b0bd09183..4741edbad934 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc
@@ -22,6 +22,13 @@ This section details the other parsers, and gives examples for how they might be
Many of these parsers are expressed the same way as xref:local-params.adoc[].
+== Auxiliary Index Join Query Parser
+
+It's similar to
+<>, but uses a lazily written sidecar index for faster joins.
+
+Details of this query parser are in the section xref:auxindexjoin-query-parser.adoc[].
+
== Block Join Query Parsers
The Block Join query parsers are used with nested documents to query for parents and/or children.