It's a bit micromanage-y, but you could add two more states to track when the button element has been entered and exited.

1. Set the modal open and set entered on mouse enter, and on mouse leave set entered false, set exited true, and then set a timeout to subsequently set open and exited false.
2. Create an .animate-unmount animation that animates only the opacity.

The unmounting animation and setTimeout should be roughly the same duration.

styles.css

.animate-mount {
  animation: mount 0.4s linear;
}

@keyframes mount {
  0% {
    opacity: 0;
    transform: translateY(50%);
  }
  100% {
    opacity: 1;
    transform: translateY(0);
  }
}

.animate-unmount {
  animation: unmount 0.5s ease-in-out;
}

@keyframes unmount {
  0% {
    opacity: 1;
  }
  100% {
    opacity: 0;
  }
}

Reactions.js

export const Actions = ({ defaultSelected = "like" }) => {
  const [selected, setSelected] = useState(defaultSelected);
  const [isOpen, setOpenTrue, setOpenFalse] = useBoolean();
  const [isEntered, setEnteredTrue, setEnteredFalse] = useBoolean();
  const [isExited, setExitedTrue, setExitedFalse] = useBoolean();

  return (
    <div className="relative rounded">
      <button
        onMouseEnter={() => {
          setOpenTrue();
          setEnteredTrue();
        }}
        onMouseLeave={() => {
          setEnteredFalse();
          setExitedTrue();
          setTimeout(() => {
            setOpenFalse();
            setExitedFalse();
          }, 500);
        }}
        className={classNames("flex items-center gap-2 rounded px-2 py-1", {
          "": isOpen
        })}
      >
        {selected && <Fragment>open modal</Fragment>}
        {isOpen && (
          <div className="absolute mt-20 left-0 flex justify-center items start h-12">
            <div
              className={classNames(
                "bg-green-700 w-24 h-36 shadow grid grid-flow-col  px-2 py-1 gap-2",
                {
                  "animate-mount": isEntered,
                  "animate-unmount": isExited
                }
              )}
            ></div>
          </div>
        )}
      </button>
    </div>
  );
};