So I thought it would be easier just to reduce the array down myself and provide that as the dataset for ChartJS, i'll leave the code below for anyone in a similar position. It makes use of a React hook to update whenever the actual dataset is updated:

/**
 * When the metric values have been updated
 */
useEffect(() => {
    /**
     * Init a new object to store the values in
     */
    let sorted = {};
    /**
     * Loop through all the metric values that were added
     */
    metricValues.forEach((value) => {
        /**
         * If we already have a copy of it in the new object
         */
        if (sorted[`${value.x}`]) {
            /**
             * Push the value onto the array
             */
            sorted[`${value.x}`] = [...sorted[`${value.x}`], value.y];
        } else {
            /**
             * Otherwise just set it as the first value
             */
            sorted[`${value.x}`] = [value.y];
        }
    });
    /**
     * Init a new array to hold the reduced data
     */
    let reducedData = [];
    /**
     * Loop through each of the now sorted values
     */
    Object.entries(sorted).forEach((dataPoint) => {
        /**
         * Get the records array, holding the values to be averaged
         */
        const dateRecords = dataPoint[1];
        /**
         * Make a sum of all the records
         */
        const recordsSum = dateRecords.reduce((a, b) => parseFloat(a) + parseFloat(b), 0);
        /**
         * Then find the avaerage
         */
        const recordsAvg = parseFloat(recordsSum / dateRecords.length).toFixed(2);
        /**
         * Push this new reduced version of the metric records to our local array
         */
        reducedData = [...reducedData, { x: `${dataPoint[0]}`, y: recordsAvg }];
    });
    /**
     * Push the complete reduced array into the local state
     */
    setReducedMetricValues(reducedData);
}, [metricValues]);