diff --git a/internal/executor/agent/agent.go b/internal/executor/agent/agent.go
index e371c52d..2c610f1d 100644
--- a/internal/executor/agent/agent.go
+++ b/internal/executor/agent/agent.go
@@ -26,7 +26,7 @@ func RetrieveBinary(
 
 	agentCacheDir := filepath.Join(cacheDir, "cirrus", "agent")
 
-	if err := os.MkdirAll(agentCacheDir, 0700); err != nil {
+	if err := os.MkdirAll(agentCacheDir, 0755); err != nil {
 		return "", err
 	}
 
@@ -78,7 +78,7 @@ func RetrieveBinary(
 	}
 
 	// Make the agent binary executable
-	if err := tmpAgentFile.Chmod(0500); err != nil {
+	if err := tmpAgentFile.Chmod(0544); err != nil {
 		return "", err
 	}
 
diff --git a/internal/executor/instance/persistentworker/pwdir/pwdir.go b/internal/executor/instance/persistentworker/pwdir/pwdir.go
index 7e2dd6b7..0730635e 100644
--- a/internal/executor/instance/persistentworker/pwdir/pwdir.go
+++ b/internal/executor/instance/persistentworker/pwdir/pwdir.go
@@ -1,6 +1,7 @@
 package pwdir
 
 import (
+	"github.com/cirruslabs/cirrus-cli/pkg/privdrop"
 	"os"
 	"path/filepath"
 )
@@ -12,5 +13,26 @@ func StaticTempDirWithDynamicFallback() (string, error) {
 		return staticTempDir, nil
 	}
 
-	return os.MkdirTemp("", "cirrus-build-")
+	// Make sure that the agent binary belongs to the privilege-dropped
+	// user and group, in case privilege dropping was requested
+	if chownTo := privdrop.ChownTo; chownTo != nil {
+		if err := os.Chown(staticTempDir, chownTo.UID, chownTo.GID); err != nil {
+			return "", err
+		}
+	}
+
+	tempDir, err := os.MkdirTemp("", "cirrus-build-")
+	if err != nil {
+		return "", err
+	}
+
+	// Make sure that the agent binary belongs to the privilege-dropped
+	// user and group, in case privilege dropping was requested
+	if chownTo := privdrop.ChownTo; chownTo != nil {
+		if err := os.Chown(tempDir, chownTo.UID, chownTo.GID); err != nil {
+			return "", err
+		}
+	}
+
+	return tempDir, nil
 }