I am trying to scrape prosthetists on the NPI public API. Currently, I am resolving a promise after 8 seconds before moving on to the next batch of 200 results and scraping them. I think this is a simplistic approach and I am curious if there is a way to track the time of the API call to finish, in this case where the console.log('The "data to append" was appended to file!), and then move on to the next iteration in the loop? Sometimes 8 seconds is enough time, and sometimes it is way too much time. What options do I have?

Here is my code:

const fs = require("fs");
const axios = require("axios");

const limit = 10000;

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

const getDoctors = async (website) => {
  const requestOptions = {
    method: "GET",

    redirect: "follow",
  };

  try {
    console.log(`scraping ${website}`);
    const { data } = await axios.get(website, requestOptions);

    await fs.readFile(
      "./customer.json",
      "utf8",
      async function (err, beepboop) {
        if (err) console.log("Oh no an error!");

        if (beepboop) {
          const json = JSON.parse(beepboop);
          json.push(data.results);

          await fs.writeFile(
            "./customer.json",
            JSON.stringify(json),
            function (err) {
              if (err) throw err;
              console.log('The "data to append" was appended to file!');
            }
          );
        } else {
          console.log("Creating file ===============================");
          await fs.writeFile(
            "./customer.json",
            JSON.stringify(data.results),
            function (err) {
              if (err) throw err;
              console.log('The "data to append" was appended to file!');
            }
          );
        }
      }
    );
  } catch (err) {
    console.log(err);
  }
};

const scrapeDoctors = async () => {
  for (let skipNumber = 0; skipNumber < limit; skipNumber += 200) {
    let url = `https://npiregistry.cms.hhs.gov/api/?version=2.1&taxonomy_description=prosthetist&limit=200&skip=${skipNumber}`;
    getDoctors(url);
    await sleep(8000);
  }
};
scrapeDoctors();