diff --git a/barbicanclient/secrets.py b/barbicanclient/secrets.py
index 844ef4aa..0286de0a 100644
--- a/barbicanclient/secrets.py
+++ b/barbicanclient/secrets.py
@@ -174,7 +174,7 @@ class Secret(SecretFormatter):
         """
         Lazy-loaded property that holds the unencrypted data
         """
-        if self._payload is None:
+        if self._payload is None and self.secret_ref is not None:
             self._fetch_payload()
         return self._payload