There are two simple steps to the problem.

First, you need to reduce the array to an object with years and values:

// this outputs
// { 2017: [8, 5], 2018: [9], 2019: [7] }
function byYear(array) {
    // take each item of an array
    return array.reduce((acc, data) => {
        // take the values of that item
        Object.entries(data.values).forEach(([year, value]) => {
            // and map all the values to years
            acc[year] = acc[year] || []
            acc[year].push(value)
        })
        return acc
    }, {})
}

The second step is just taking averages:

function average(object) {
    const averages = {}
    for (let key in object) {
        averages[key] = object[key].reduce((sum, value) => sum + value) / object[key].length 
    }
    return averages
}

And now you put them together:

average(byYear(input))

In here, the input is the filtered array. As a whole snippet:

function byYear(array) {
    return array.reduce((acc, data) => {
        Object.entries(data.values).forEach(([year, value]) => {
            acc[year] = acc[year] || []
            acc[year].push(value)
        })
        return acc
    }, {})
}

function average(object) {
    const averages = {}
    for (let key in object) {
        averages[key] = object[key].reduce((sum, value) => sum + value) / object[key].length 
    }
    return averages
}

const output = average(byYear([{
  course: "math",
  id: 4,
  values: {
    2017: 8,
    2018: 9
  }
}, {
  course: "math",
  id: 4,
  values: {
    2017: 5,
    2019: 7
  }
}]))

console.log(output)

The problem with your current code lies in how you build the reformattedArray variable. First, notice that your map function implicitly returns undefined whenever that year is missing from the current object:

let reformattedArray = mathid1.map(obj => {
  if (obj["values"][year]) {
    return obj["values"][year]
  }

  // There is an implicit return undefined, right here...
})

When you use the array .map method, every item of the array will be replaced by the return value of the map function. In the case that the year is not present, it will not go into the if block, and so it implicitly returns undefined upon reaching the end of the function.

So, ultimately all you have to do is remove the undefined entries from this array, and your code will work as-is.

One way to do that is to just use .filter(Boolean) on the array, which removes any falsey entries (which undefined is). Eg:

let reformattedArray = mathid1.map(obj => {
  /* code here */
}).filter(Boolean); // Note the filter here...

Here is your snippet with that modification:

const testObject = [{
    id: 4,
    course: "math",
    values: {
      2017: 8,
      2018: 9
    }
  },
  {
    id: 5,
    course: "English",
    values: {
      2017: 8,
      2018: 9
    }
  },
  {
    id: 4,
    course: "math",
    values: {
      2017: 5,
      2019: 7
    }
  },
  {
    id: 4,
    course: "english",
    values: {
      2017: 5,
      2019: 7
    }
  },
]
//First I filter out the id 4 and course Math
const mathid1 = testObject.filter((e) => e.id === 4 && e.course === "math");
//I than find all the different years
const ArrayOfAllYears = []
mathid1.map((element) => {
  ArrayOfAllYears.push(Object.keys(element.values));
})
//I here find all the different years
const withDuplicates = ArrayOfAllYears.reduce(function(arrayOne, arrayTwo) {
  return arrayOne.concat(arrayTwo);
}, []);

const withoutDuplicates = Array.from(new Set(withDuplicates));
//Here I just create the calculate average function
const Result = {}
const calculateAverage = (array) => {
  const sum = array.reduce((a, b) => a + b);
  return sum / array.length;
};
const newObj = {}
withoutDuplicates.map((year) => {
  let reformattedArray = mathid1.map(obj => {
    if (obj["values"][year]) {
      return obj["values"][year]
    }
  }).filter(Boolean)
  newObj[year] = calculateAverage(reformattedArray)
})
console.log(newObj)
// I want to calculate the average of the mathid1 values and return it on a Object like {2017:..,2018..}

1. Group items by year.
2. Calculate average.

const items=[{
  course: "math",
  id: 4,
  values: {
    2017: 8,
    2018: 9
  }
}, {
  course: "math",
  id: 4,
  values: {
    2017: 5,
    2019: 7
  }
}]

const groupedValues=items.reduce((groupedValues,item)=>{
  Object.entries(item.values).forEach(([year,value])=>{
      if(groupedValues[year]){
        groupedValues[year]={value:groupedValues[year].value+value,items:groupedValues[year].items+1};
      } else {
        groupedValues[year]={value,items:1};
      }
  });
  return groupedValues;
},{})

console.log(groupedValues);

const result = Object.entries(groupedValues).reduce((result,item)=>{
  result[item[0]]=item[1].value/item[1].items;
  return result;
},{})

console.log(result);