For styling I suggest conditionally applying a classname when you want to display the back-to-top button. You will also want to move the adding (and removing) of the window scroll listener into an useEffect hook. The useEffect hook should add the event listener and return a cleanup function to remove it when the ScrollTop component unmounts. Provide false as the third argument so you use passive listeners.

const ScrollTop = () => {
  const [visible, setVisible] = useState(false);

  // Show Button After Scrolling Down More than 500px
  const toggleVisible = () => {
    if (
      document.body.scrollTop > 500 ||
      document.documentElement.scrollTop > 500
    ) {
      setVisible(true);
    } else {
      setVisible(false);
    }
  };

  useEffect(() => {
    // Listen for Scrolling Event
    window.addEventListener("scroll", toggleVisible, false);
    return () => {
      window.removeEventListener("scroll", toggleVisible, false);
    }
  }, []);

  // Get Back Top when Clicked
  const handleScroll = () => {
    window.scrollTo({
      top: 0,
    });
  };

  return (
    <button
      id="back-to-top"
      className={visible ? "back-to-top-visible" : null}
      onClick={handleScroll}
      title="Go To Top"
    >
      <i className="fas fa-chevron-circle-up"></i>
    </button>
  );
};

CSS - How you want to transition is up to you, but the basic starting point is to define a transition rule with property/ies to transition, a duration, easing/timing function, and delay. For the example I chose to transition the scaling. Start with an initial scaling value of 0, and when the back-to-top-visible classname is applied end with a scaling value of 1.

Using CSS transitions

#back-to-top {
  background: none;
  background-repeat: no-repeat;
  border: none;
  cursor: pointer;
  overflow: hidden;
  outline: none;
  position: fixed;
  bottom: 30px;
  right: 20px;
  color: rgb(255, 51, 0);
  z-index: 99;
  font-size: 2.5rem;
  transform: scale(0);
  transition: all 0.5s ease-in-out;
}

#back-to-top:hover {
  color: rgb(255, 0, 140);
}

#back-to-top.back-to-top-visible {
  transform: scale(1);
}

All you need is to target the button in your CSS and add transition like this:

#back-to-top {
   transition: all 0.2s linear;
}

or you can try this solution where you can conditionally render class.

CSS:

#back-to-top {
  background: none;
  background-repeat: no-repeat;
  border: none;
  cursor: pointer;
  overflow: hidden;
  outline: none;
  position: fixed;
  bottom: 30px;
  right: 20px;
  color: rgb(255, 51, 0);
  z-index: 99;
  font-size: 2.5rem;
  transition: all 0.6s linear;
}
.hide-class {
  opacity: 0;
  z-index: -100;
}

.show-class {
  opacity: 1;
}

#back-to-top:hover {
  color: rgb(255, 0, 140);
}

import React, {useState} from "react";
import "./style.css";

ScrollTop:

const App = () => {
  const [visible, setVisible] = useState(false);

  // Show Button After Scrolling Down More than 500px
  const toggleVisible = () => {
    if (
      document.body.scrollTop > 500 ||
      document.documentElement.scrollTop > 500
    ) {
      setVisible(true);
    } else {
      setVisible(false);
    }
  };

  // Listen for Scrolling Event
  window.addEventListener("scroll", toggleVisible);

  // Get Back Top when Clicked
  const handleScroll = () => {
    window.scrollTo({
      top: 0,
    });
  };

  return (
    <React.Fragment>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
      <p>Content</p><p>Content</p>
    <button
      id="back-to-top"
      className={`${visible ? 'show-class': 'hide-class'}`}
      onClick={handleScroll}
      title="Go To Top"
    >
      Go to top
    </button>
      </React.Fragment>
  );
};

export default App;

Stackblitz link