Rest Paramanter syntax (...) is used when you are handling with dynamic number of arguments. Here you are passing only one argument to the function. Remove the ... from the function param and your function will work as expected.

const scores = [35, 68, 70, 38];
const scores1 = [89, 42, 33];
function gradingStudents(grades) {
  const newScores = grades.map(function (grade) {
    if (grade + 2 >= 40 && grade % 5 >= 3) {
      return grade % 5 == 3 ? grade + 2 : grade + 1;
    } else if (grade + 2 >= 40 && grade % 5 < 3) {
      return grade;
    } else {
      return grade;
    }
  });
  return newScores;
}
console.log(gradingStudents(scores1));

If you are handling the function using Rest Paramanter syntax as you did now, the parameter in the function written as ...grades will combine all the paramaters in the function to a single array. Which means your paramater scores1 will be there inside the grades array as in the below console.

const scores1 = [89, 42, 33];
function gradingStudents(...grades) {
  // grades will be an array with scores1 as a sigle element in 0th index
  console.log(grades)
}
gradingStudents(scores1);

Looping through grades in the above condition, the each node will be an array which will not satisfy any of the if statement, hence it will return the array itself. Thats why you are getting the output as an array with the parementer as the single node.

since you are returning grade if the first condition didn't meet. The rest operator is not needed. You can write it as follow:

function gradingStudents(grades) {
        const newScores = grades.map(function (grade) {
            if (grade + 2 >= 40 && grade % 5 >= 3) {
                return grade % 5 == 3 ? grade + 2 : grade + 1;
            }else {
                return grade;
            }
       });
       return newScores;
  }

You can use forEach also

const scores = [35, 68, 70, 38];
const scores1 = [89, 42, 33];

function gradingStudents(grades) {
   let newScore = [];
   grades.forEach(function (grade) {
    if (grade + 2 >= 40 && grade % 5 >= 3) {
      grade = grade % 5 == 3 ? grade + 2 : grade + 1;
      newScore.push(grade);
    } else if (grade + 2 >= 40 && grade % 5 < 3) {
      newScore.push(grade);
    } else {
      newScore.push(grade);
    }
  });
  return newScore;
}

console.log(gradingStudents(scores));
console.log(gradingStudents(scores1));