Replace your ~/.vnc/xstartup file with the following content:
#!/bin/sh unset SESSION_MANAGER exec /etc/X11/xinit/xinitrc[ -x /etc/vnc/xstartup ] && exec /etc/vnc/xstartup [ -r $HOME/.Xresources ] && xrdb $HOME/.Xresources xsetroot -solid grey vncconfig -iconic & xterm -geometry 80x24+10+10 -ls -title "$VNCDESKTOP Desktop" & startxfce4