diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7178e152..3a3c9d64 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -539,7 +539,8 @@ jobs:
             gh release download --repo "jay0lee/cryptography-wheels" --pattern "*win_arm64.whl" --clobber
             "$PYTHON" -m pip install cryptography-*.whl
           fi
-          "$PYTHON" -m pip install --upgrade ..[yubikey]
+          "$PYTHON" -m pip install yubikey-manager --no-deps
+          "$PYTHON" -m pip install ..[yubikey]
 
       - name: Install PyInstaller
         if: matrix.goal == 'build'