diff --git a/src/Psalm/Config.php b/src/Psalm/Config.php
index 5afb8e28b..92742090b 100644
--- a/src/Psalm/Config.php
+++ b/src/Psalm/Config.php
@@ -633,6 +633,14 @@ class Config
         return $config;
     }
 
+    /**
+     * Computes the hash to use for a cache folder from CLI flags and from the config file's xml contents
+     */
+    public function computeHash(): string
+    {
+        return sha1($this->hash . ':' . $this->level);
+    }
+
     /**
      * Creates a new config object from an XML string
      * @param  string|null      $current_dir Current working directory, if different to $base_dir
diff --git a/src/Psalm/Internal/Provider/ClassLikeStorageCacheProvider.php b/src/Psalm/Internal/Provider/ClassLikeStorageCacheProvider.php
index 6fa98a4f6..3ad117463 100644
--- a/src/Psalm/Internal/Provider/ClassLikeStorageCacheProvider.php
+++ b/src/Psalm/Internal/Provider/ClassLikeStorageCacheProvider.php
@@ -65,7 +65,7 @@ class ClassLikeStorageCacheProvider
             $this->modified_timestamps .= ' ' . filemtime($dependent_file_path);
         }
 
-        $this->modified_timestamps .= $this->config->hash;
+        $this->modified_timestamps .= $this->config->computeHash();
     }
 
     public function writeToCache(ClassLikeStorage $storage, ?string $file_path, ?string $file_contents): void
diff --git a/src/Psalm/Internal/Provider/FileReferenceCacheProvider.php b/src/Psalm/Internal/Provider/FileReferenceCacheProvider.php
index d03e42cea..34aff48ae 100644
--- a/src/Psalm/Internal/Provider/FileReferenceCacheProvider.php
+++ b/src/Psalm/Internal/Provider/FileReferenceCacheProvider.php
@@ -54,8 +54,9 @@ class FileReferenceCacheProvider
 
     public function hasConfigChanged() : bool
     {
-        $has_changed = $this->config->hash !== $this->getConfigHashCache();
-        $this->setConfigHashCache($this->config->hash);
+        $new_hash = $this->config->computeHash();
+        $has_changed = $new_hash !== $this->getConfigHashCache();
+        $this->setConfigHashCache($new_hash);
         return $has_changed;
     }
 
diff --git a/src/Psalm/Internal/Provider/FileStorageCacheProvider.php b/src/Psalm/Internal/Provider/FileStorageCacheProvider.php
index f6d3cf882..ba91df228 100644
--- a/src/Psalm/Internal/Provider/FileStorageCacheProvider.php
+++ b/src/Psalm/Internal/Provider/FileStorageCacheProvider.php
@@ -66,7 +66,7 @@ class FileStorageCacheProvider
             $this->modified_timestamps .= ' ' . filemtime($dependent_file_path);
         }
 
-        $this->modified_timestamps .= $this->config->hash;
+        $this->modified_timestamps .= $this->config->computeHash();
     }
 
     public function writeToCache(FileStorage $storage, string $file_contents): void