[SYCL] Implement eviction for in-memory program cache (#16062)

Fixes: CMPLRLLVM-27640, #2517 The PR implements LRU cache eviction policy for in-memory program caches. The high-level idea is to store programs in a linked-list, called eviction list. When the program is first added to the cache, it is also added to the eviction list. When a program is fetched from cache, we move the program to the end of the eviction list. So, that the programs at the beginning of the eviction list are always least recently used. When adding a new program to cache, we check if the size of the program cache exceeds the threshold, if so, we evict the program from cache and corresponding kernels from Kernel and fast kernel cache. This PR also adds a new environment variable, `SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD` that user can use to control the size of in-memory cache. By default, cache eviction is disabled.
intel · Nov 20, 2024 · 0e2094d · 0e2094d
1 parent f0899ff
commit 0e2094d
Show file tree

Hide file tree

Showing 9 changed files with 708 additions and 11 deletions.
diff --git a/sycl/doc/EnvironmentVariables.md b/sycl/doc/EnvironmentVariables.md
@@ -14,6 +14,7 @@ compiler and runtime.
 | `SYCL_CACHE_DISABLE_PERSISTENT (deprecated)` | Any(\*) | Has no effect. |
 | `SYCL_CACHE_PERSISTENT` | Integer | Controls persistent device compiled code cache. Turns it on if set to '1' and turns it off if set to '0'. When cache is enabled SYCL runtime will try to cache and reuse JIT-compiled binaries. Default is off. |
 | `SYCL_CACHE_IN_MEM` | '1' or '0' | Enable ('1') or disable ('0') in-memory caching of device compiled code. When cache is enabled SYCL runtime will try to cache and reuse JIT-compiled binaries. Default is '1'. |
+| `SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD` | Positive integer  | `SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD` accepts an integer that specifies the maximum size of the in-memory program cache in bytes. Eviction is performed when the cache size exceeds the threshold. The default value is 0 which means that eviction is disabled.  |
 | `SYCL_CACHE_EVICTION_DISABLE` | Any(\*) | Switches persistent cache eviction off when the variable is set. |
 | `SYCL_CACHE_MAX_SIZE` | Positive integer | Persistent cache eviction is triggered once total size of cached images exceeds the value in megabytes (default - 8 192 for 8 GB). Set to 0 to disable size-based cache eviction. |
 | `SYCL_CACHE_THRESHOLD` | Positive integer | Persistent cache eviction threshold in days (default value is 7 for 1 week). Set to 0 for disabling time-based cache eviction. |

diff --git a/sycl/source/detail/config.def b/sycl/source/detail/config.def
@@ -27,6 +27,8 @@ CONFIG(SYCL_HOST_UNIFIED_MEMORY, 1, __SYCL_HOST_UNIFIED_MEMORY)
 // 260 (Windows limit) - 12 (filename) - 84 (cache directory structure)
 CONFIG(SYCL_CACHE_DIR, 164, __SYCL_CACHE_DIR)
 CONFIG(SYCL_CACHE_TRACE, 4, __SYCL_CACHE_TRACE)
+CONFIG(SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD, 16,
+       __SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD)
 CONFIG(SYCL_CACHE_DISABLE_PERSISTENT, 1, __SYCL_CACHE_DISABLE_PERSISTENT)
 CONFIG(SYCL_CACHE_PERSISTENT, 1, __SYCL_CACHE_PERSISTENT)
 CONFIG(SYCL_CACHE_EVICTION_DISABLE, 1, __SYCL_CACHE_EVICTION_DISABLE)

diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
@@ -756,6 +756,56 @@ template <> class SYCLConfig<SYCL_CACHE_TRACE> {
   }
 };
 
+// SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD accepts an integer that specifies
+// the maximum size of the in-memory Program cache.
+// Cache eviction is performed when the cache size exceeds the threshold.
+// The thresholds are specified in bytes.
+// The default value is "0" which means that eviction is disabled.
+template <> class SYCLConfig<SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD> {
+  using BaseT = SYCLConfigBase<SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD>;
+
+public:
+  static int get() { return getCachedValue(); }
+  static void reset() { (void)getCachedValue(true); }
+
+  static int getProgramCacheSize() { return getCachedValue(); }
+
+  static bool isProgramCacheEvictionEnabled() {
+    return getProgramCacheSize() > 0;
+  }
+
+private:
+  static int getCachedValue(bool ResetCache = false) {
+    const auto Parser = []() {
+      const char *ValStr = BaseT::getRawValue();
+
+      // Disable eviction by default.
+      if (!ValStr)
+        return 0;
+
+      int CacheSize = 0;
+      try {
+        CacheSize = std::stoi(ValStr);
+        if (CacheSize < 0)
+          throw INVALID_CONFIG_EXCEPTION(BaseT, "Value must be non-negative");
+      } catch (...) {
+        std::string Msg = std::string{
+            "Invalid input to SYCL_IN_MEM_CACHE_EVICTION_THRESHOLD. Please try "
+            "a positive integer."};
+        throw exception(make_error_code(errc::runtime), Msg);
+      }
+
+      return CacheSize;
+    };
+
+    static auto EvictionThresholds = Parser();
+    if (ResetCache)
+      EvictionThresholds = Parser();
+
+    return EvictionThresholds;
+  }
+};
+
 #undef INVALID_CONFIG_EXCEPTION
 
 } // namespace detail