diff --git a/.github/workflows/cmake_build.yml b/.github/workflows/cmake_build.yml
index abf194ec1..2cf84dd68 100644
--- a/.github/workflows/cmake_build.yml
+++ b/.github/workflows/cmake_build.yml
@@ -200,7 +200,6 @@ jobs:
     - name: "Authenticate CI to Github Packages"
       if: github.repository_owner == 'tenacityteam'
       run: |
-        rm ./nuget.config
         nuget sources add -Name tenacityteam_github_auto -Source https://nuget.pkg.github.com/tenacityteam/index.json -Username tenacityteam -Password ${{ secrets.GITHUB_TOKEN }} -StorePasswordInClearText -ForceEnglishOutput -NonInteractive
         nuget setapikey ${{ secrets.GITHUB_TOKEN }} -Source tenacityteam_github_auto -ForceEnglishOutput -NonInteractive
         echo "VCPKG_BINARY_SOURCES=clear;nuget,tenacityteam_github_auto,readwrite;" >> ${GITHUB_ENV}
diff --git a/.gitignore b/.gitignore
index ca58ea661..de18975d4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -219,3 +219,6 @@ vcpkg_installed
 
 # Flatpak
 !packaging/**
+
+# Prevent accidentally uploading complete local PAT
+nuget.config
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7da780ddf..8ea797684 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -105,7 +105,7 @@ if( VCPKG )
   set( ENV{VCPKG_DISABLE_METRICS} true )
 
   if( NOT DEFINED ENV{VCPKG_BINARY_SOURCES} )
-    set( ENV{VCPKG_BINARY_SOURCES} "clear;nuget,tenacityteam_github_auto,read;" )
+    set( ENV{VCPKG_BINARY_SOURCES} "clear;default;nuget,tenacityteam_github_auto,read;" )
   endif()
 
   set( ENV{VCPKG_FEATURE_FLAGS} "-compilertracking,manifests,registries,versions" )
@@ -144,6 +144,36 @@ if( VCPKG )
     endif()
   endif()
 
+  if( NOT DEFINED ENV{CI} AND NOT EXISTS ${CMAKE_SOURCE_DIR}/nuget.config AND (EXISTS ${VCPKG_ROOT}/vcpkg OR EXISTS ${VCPKG_ROOT}/vcpkg.exe) )
+
+    execute_process(
+      COMMAND vcpkg fetch nuget
+      WORKING_DIRECTORY ${VCPKG_ROOT}
+      OUTPUT_VARIABLE NUGET_EXECUTABLE
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+    message( STATUS "Automatically setting Nuget api key to allow nuget cache usage locally" )
+
+    file( READ ${CMAKE_SOURCE_DIR}/nuget.config.in NUGET_CONFIG_CONTENT )
+    file( WRITE ${CMAKE_SOURCE_DIR}/nuget.config ${NUGET_CONFIG_CONTENT} )
+
+    set( GITHUB_PACKAGES_TOKEN_SUFFIX "ZdXoXftxDTV20UjAmIL2Mw95kckhbo0hmiYz" )
+
+    execute_process(
+      COMMAND "${NUGET_EXECUTABLE}" sources add -Name tenacityteam_github_auto -Source https://nuget.pkg.github.com/tenacityteam/index.json -Config nuget.config -Username tenacityteamci  -Password ghp_${GITHUB_PACKAGES_TOKEN_SUFFIX} -StorePasswordInClearText -ForceEnglishOutput -NonInteractive
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    )
+
+    execute_process(
+      COMMAND "${NUGET_EXECUTABLE}" setapikey -Source tenacityteam_github_auto -Config nuget.config ghp_${GITHUB_PACKAGES_TOKEN_SUFFIX}
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE SETAPI_KEY_RESULT
+      ERROR_VARIABLE SETAPI_KEY_RESULT
+    )
+
+  endif()
+
   if( NOT DEFINED VCPKG_OVERLAY_PORTS )
     set( VCPKG_OVERLAY_PORTS "${VCPKG_ROOT}/overlay/ports" )
   endif()
diff --git a/nuget.config b/nuget.config
deleted file mode 100644
index f086ec78d..000000000
--- a/nuget.config
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-
-    <config>
-        <add key="defaultPushSource" value="https://nuget.pkg.github.com/tenacityteam/index.json" />
-    </config>
-
-    <packageSources>
-        <add key="tenacityteam_github_auto" value="https://nuget.pkg.github.com/tenacityteam/index.json" />
-    </packageSources>
-
-
-    <packageSourceCredentials>
-        <tenacityteam_github_auto>
-            <add key="Username" value="tenacityteamci" />
-            <add key="Password" value="Z2hwX1pkWG9YZnR4RFRWMjBVakFtSUwyTXc5NWtja2hibzBobWlZeg==" />
-        </tenacityteam_github_auto>
-    </packageSourceCredentials>
-
-    <apikeys>
-        <add key="https://nuget.pkg.github.com/tenacityteam/index.json" value="AQAAANCMnd8BFdERjHoAwE/Cl+sBAAAAUqERagW7+Eu+K0Mct8Lx1QAAAAACAAAAAAAQZgAAAAEAACAAAABgLxr6r/iMxsCDAsoK6EMR0pQlfBfLa2ETAzyCYf8eGgAAAAAOgAAAAAIAACAAAAD+yqdTYe/0Sy9Gmwn5sdaXDh1Bb2I7e9MRDX5HnS1mDDAAAAAU2YvFeiD/IhDcj84qHDE2QUcGsiVCRWecyHvUuedqqMxp5fHj65ZsXj5TPpuYafhAAAAAzx0UqJezLSUsHGwGwctzQvrz0CDNdbUHYXyAI/GGh6lbGv0k3yGJTpF9LFoUvOGQz1mvneSE0NHZAjkMGLuMAw==" />
-    </apikeys>
-
-</configuration>
diff --git a/nuget.config.in b/nuget.config.in
new file mode 100644
index 000000000..f985ec7c0
--- /dev/null
+++ b/nuget.config.in
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+    <config>
+        <add key="defaultPushSource" value="https://nuget.pkg.github.com/tenacityteam/index.json"/>
+    </config>
+</configuration>