I am going to make the assumption that you've calculated the scale factor correctly - I'd either have to think really critically about the scaleFactorX and scaleFactorY idea or just play with code (of which I am too lazy to do at this moment), but you may only need to check for 1 of them for scaling:

function rescale_canvas_if_needed() {
  var optimal_dimensions = [$(".canvasWrapper").outerWidth(), $(".canvasWrapper").outerHeight()];

  // add a rectangle the size of the current canvas and center it
  rect = new fabric.Rect({
    left: 0,
    top: 0,
    width: canvas.width,
    height: canvas.height,
    angle: 0,
    fill: 'rgba(255,255,255,0)',
    transparentCorners: true
    });
  canvas.add(rect);
  rect.center();

  // get an active selection of all objects

  canvas.discardActiveObject();
  var sel = new fabric.ActiveSelection(canvas.getObjects(), {canvas: canvas});
  canvas.setActiveObject(sel);
  canvas.requestRenderAll();

  // scale your canvas

  var scaleFactorX = window.innerWidth / optimal_dimensions[0];
  var scaleFactorY = window.innerHeight / optimal_dimensions[1];
  if (scaleFactorX < scaleFactorY && scaleFactorX < 1) {
    canvas.setWidth(optimal_dimensions[0] * scaleFactorX);
    canvas.setHeight(optimal_dimensions[1] * scaleFactorX);
    canvas.setZoom(scaleFactorX);
  } else if (scaleFactorX > scaleFactorY && scaleFactorY < 1) {
    canvas.setWidth(optimal_dimensions[0] * scaleFactorY);
    canvas.setHeight(optimal_dimensions[1] * scaleFactorY);
    canvas.setZoom(scaleFactorY);
  } else {
    canvas.setWidth(optimal_dimensions[0]);
    canvas.setHeight(optimal_dimensions[1]);
    canvas.setZoom(1);
  }
  canvas.calcOffset();

  // scale the active selection to the new width

  sel.scaleToWidth(canvas.width);

  // center the selection

  sel.center();

  // deselect the active selection

  canvas.discardActiveObject();

  // delete the rectangle

  canvas.remove(rect);

  canvas.requestRenderAll();
}


function handle_resize() {
  // I don't think you should have to hide or show the canvas-container
  $(".canvas-container").hide();
  rescale_canvas_if_needed();
  $(".canvas-container").show();

}

This will discard any active selections on the page before resizing it - maybe thats how you will want it, but you could get the active selection before resizing everything, then once it is done with all the resizing, reselect that active selection.

Warning: I didn't test this, so it may have a bug, invalid syntax, or other problems within that nature.

Let me know if you can't past another hurtle, given this bit of code.