I have a gameobject that moves forward constantly, and I'm controlling it with the mouse on the X axis using the script below, and the camera is following this object.

The issue I have is when letting the camera follow it on the X axis, the object's movement become weird (because of the WorldToScreenPoint position change I think), but it works perfectly when freezing the camera's X position.

My question is how can I make the camera follow this object on the X axis without affecting the movement?

if ((Input.GetMouseButtonDown(0) || Input.touchCount > 0 && Input.GetTouch(0).phase == TouchPhase.Began))
    {
        screenPoint = Camera.main.WorldToScreenPoint(transform.position);
        offset = transform.position - Camera.main.ScreenToWorldPoint(new Vector3(Input.mousePosition.x, screenPoint.y, screenPoint.z));
    }

    if (((Input.GetMouseButton(0) && !Input.GetMouseButtonDown(0)) || ((Input.touchCount > 0 && Input.GetTouch(0).phase == TouchPhase.Moved)) && !(Input.touchCount > 0 && Input.GetTouch(0).phase == TouchPhase.Began)))
    {
        Vector3 curScreenPoint = new Vector3(Input.mousePosition.x, screenPoint.y, screenPoint.z);

        Vector3 curPosition = new Vector3((Camera.main.ScreenToWorldPoint(curScreenPoint).x + offset.x), 0, 0);

        float newX = Mathf.Clamp(curPosition.x, -2, 2);

        transform.position = Vector3.Lerp(transform.position, new Vector3(newX, transform.position.y, transform.position.z), 25 * Time.deltaTime);
    }