diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c
index 0b7e7a9..e5b616c 100644
--- a/fs/overlayfs/copy_up.c
+++ b/fs/overlayfs/copy_up.c
@@ -354,7 +354,8 @@ int ovl_set_origin(struct ovl_fs *ofs, struct dentry *dentry,
 				 fh ? fh->fb.len : 0, 0);
 	kfree(fh);
 
-	return err;
+	/* Ignore -EPERM from setting "user.*" on symlink/special */
+	return err == -EPERM ? 0 : err;
 }
 
 /* Store file handle of @upper dir in @index dir entry */