Have you added the CGC driver, and started the Main Oscillator before calling R_CAN_Open()?

err = R_CGC_Open (&g_cgc0_ctrl, &g_cgc0_cfg);
if (FSP_SUCCESS != err)
{
__BKPT(0); //TODO better error handling
}
err = R_CGC_ClockStart (&g_cgc0_ctrl, CGC_CLOCK_MAIN_OSC, NULL);
if (FSP_SUCCESS != err)
{
__BKPT(0); //TODO better error handling
}

do
{
err = R_CGC_ClockCheck(&g_cgc0_ctrl, CGC_CLOCK_MAIN_OSC);
}while(FSP_ERR_NOT_STABILIZED == err);
if (FSP_SUCCESS != err)
{
__BKPT(0); //TODO better error handling
}

/* Initialize CAN module */
err = R_CAN_Open(&g_can0_ctrl, &g_can0_cfg);
/* Error trap */
if(FSP_SUCCESS != err)
{
__BKPT(0); //TODO better error handling
}