IGNITE-29032 Improvement of the page eviction mechanism for in-memory

modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java

@@ -18,7 +18,6 @@
 
 import java.io.Serializable;
 import org.apache.ignite.DataRegionMetrics;
-import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.mem.MemoryAllocator;
 import org.apache.ignite.mxbean.MetricsMxBean;
@@ -346,9 +345,9 @@ public DataRegionConfiguration setEvictionThreshold(double evictionThreshold) {
      * Specifies the minimal number of empty pages to be present in reuse lists for this data region.
      * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of
      * (key, value) pair is slightly larger than page size / 2.
-     * Increase this parameter if cache can contain very big entries (total size of pages in this pool should be enough
-     * to contain largest cache entry).
-     * Increase this parameter if {@link IgniteOutOfMemoryException} occurred with enabled page eviction.
+     * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool,
+     * it is no longer required to increase this parameter up to the size of the largest cache entry;
+     * it may be kept at its default as the steady-state reserve of empty pages.
      *
      * @return Minimum number of empty pages in reuse list.
      */
@@ -360,9 +359,9 @@ public int getEmptyPagesPoolSize() {
      * Specifies the minimal number of empty pages to be present in reuse lists for this data region.
      * This parameter ensures that Ignite will be able to successfully evict old data entries when the size of
      * (key, value) pair is slightly larger than page size / 2.
-     * Increase this parameter if cache can contain very big entries (total size of pages in this pool should be enough
-     * to contain largest cache entry).
-     * Increase this parameter if {@link IgniteOutOfMemoryException} occurred with enabled page eviction.
+     * Since size-aware eviction automatically frees additional pages when the inserted row is larger than this pool,
+     * it is no longer required to increase this parameter up to the size of the largest cache entry;
+     * it may be kept at its default as the steady-state reserve of empty pages.
      *
      * @param emptyPagesPoolSize Empty pages pool size.
      * @return {@code this} for chaining.

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java

@@ -201,6 +201,24 @@ public interface GridCacheEntryEx {
     public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter,
         boolean evictOffheap) throws IgniteCheckedException;
 
+    /**
+     * Same as {@link #evictInternal(GridCacheVersion, CacheEntryPredicate[], boolean)}, but acquires the entry lock
+     * non-blockingly when {@code tryLock} is {@code true}, returning {@code false} (instead of blocking) if the entry
+     * lock is contended. Used by size-aware page eviction which may run while the current thread already holds other
+     * entry locks, to avoid a lock-ordering deadlock. The default implementation uses the blocking variant.
+     *
+     * @param obsoleteVer Version for eviction.
+     * @param filter Optional filter.
+     * @param evictOffheap Evict offheap value flag.
+     * @param tryLock {@code true} to acquire the entry lock non-blockingly (skip contended entries).
+     * @return {@code True} if entry could be evicted.
+     * @throws IgniteCheckedException In case of error.
+     */
+    public default boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter,
+        boolean evictOffheap, boolean tryLock) throws IgniteCheckedException {
+        return evictInternal(obsoleteVer, filter, evictOffheap);
+    }
+
     /**
      * This method should be called each time entry is marked obsolete
      * other than by calling {@link #markObsolete(GridCacheVersion)}.

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java

@@ -3659,6 +3659,10 @@ protected void removeValue() throws IgniteCheckedException {
      */
     private void ensureFreeSpace() throws IgniteCheckedException {
         // Deadlock alert: evicting data page causes removing (and locking) all entries on the page one by one.
+        // This entry-level eviction is invoked only while NOT holding this entry's lock (all call sites run before
+        // lockEntry()). The separately invoked size-aware path (see RowStore.addRow ->
+        // IgniteCacheDatabaseSharedManager#ensureFreeSpaceForInsert) runs while the lock IS held, so it relies on
+        // the non-blocking tryLockEntry(0) inside evictInternal to avoid a lock-ordering deadlock.
         assert !lock.isHeldByCurrentThread();
 
         cctx.shared().database().ensureFreeSpace(cctx.dataRegion());
@@ -3687,11 +3691,28 @@ private <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
         boolean evictOffheap)
         throws IgniteCheckedException {
 
+        return evictInternal(obsoleteVer, filter, evictOffheap, false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evictInternal(
+        GridCacheVersion obsoleteVer,
+        @Nullable CacheEntryPredicate[] filter,
+        boolean evictOffheap,
+        boolean tryLock)
+        throws IgniteCheckedException {
+
         boolean marked = false;
 
         try {
             if (F.isEmptyOrNulls(filter)) {
-                lockEntry();
+                // With tryLock=true (size-aware eviction running while the current thread already holds entry locks)
+                // the lock is acquired non-blockingly: a contended entry is skipped (returning false) rather than
+                // blocking, which prevents a lock-ordering deadlock between concurrent evictions. The eviction tracker
+                // will then pick another page. For all other paths (tryLock=false) the original
+                // blocking lockEntry() is preserved.
+                if (!lockEntry(tryLock))
+                    return false;
 
                 try {
                     if (evictionDisabled()) {
@@ -3728,7 +3749,8 @@ private <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                 while (true) {
                     GridCacheVersion v;
 
-                    lockEntry();
+                    if (!lockEntry(tryLock))
+                        return false;
 
                     try {
                         v = ver;
@@ -3740,7 +3762,8 @@ private <K, V> CacheEntryImplEx<K, V> wrapVersionedWithValue() {
                     if (!cctx.isAll(/*version needed for sync evicts*/this, filter))
                         return false;
 
-                    lockEntry();
+                    if (!lockEntry(tryLock))
+                        return false;
 
                     try {
                         if (evictionDisabled()) {
@@ -4182,6 +4205,23 @@ private int extrasSize() {
         lock.lock();
     }
 
+    /**
+     * Acquires the entry lock either blocking ({@code tryLock == false}) or non-blockingly with an immediate
+     * {@code tryLock(0)} ({@code tryLock == true}). Used by {@link #evictInternal} to let size-aware
+     * eviction skip contended entries instead of blocking, avoiding a lock-ordering deadlock.
+     *
+     * @param tryLock {@code true} to acquire the lock non-blockingly.
+     * @return {@code true} if the lock was acquired (always {@code true} when {@code tryLock == false}).
+     */
+    private boolean lockEntry(boolean tryLock) {
+        if (tryLock)
+            return !lock.isHeldByCurrentThread() && tryLockEntry(0);
+
+        lockEntry();
+
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean tryLockEntry(long timeout) {
         try {