The reliable way is to traverse the parent tree checking getComputedStyle.

I'll assume you have the selected element(s) already.

function isBold(_element) {
  var element = _element;
  while (element) {
    var style = getComputedStyle(element).fontWeight;
    if (Number(fontWeight) > 400 || fontWeight === 'bold' || fontWeight === 'bolder') {
      return true;
    }
    element = element.parentNode;
  }
  return false;
}

jQuery(function($) {
    $('.embolden').click(function(){
        if(selectionIsBold()){
          alert('bold');
        }
        else {
          alert('not bold');
        }
    });
});

function selectionIsBold() {
    var isBold = false;
    if (document.queryCommandState) {
        isBold = document.queryCommandState("bold");
    }
    return isBold;
}

.bold {
    font-weight: bold;
}

<div contenteditable="true" class="textEditor">Some <span class="bold">random </span>text.</div>
<a href="#" class="embolden">Is Bold Text</a>
<script
  src="https://code.jquery.com/jquery-3.2.1.min.js"
  integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
  crossorigin="anonymous"></script>

Highlight the text and click on the link.

Just call isSelectionBold() :)

Demo on CodePen

function isSelectionBold() {
  var sel;
  if (window.getSelection) { 
    sel = window.getSelection(); 
  } 
  else if (document.getSelection) { 
    sel = document.getSelection(); 
  }
  
    var raw_html = getSelectionAsHtml();
  
  // This is if nothing is selected
  if(raw_html==="") return false;

  var tempDiv = document.createElement('div');
  tempDiv.innerHTML = raw_html;
    
  var is_bold_nodes = []
    for (var node of tempDiv.childNodes) {
    var tags = [node.nodeName.toLowerCase()];
        
    // This covers selection that are inside bolded characters
    while(tags.includes("#text")) {
      var start_tag = sel.anchorNode.parentNode.nodeName.toLowerCase();
      var end_tag = sel.focusNode.parentNode.nodeName.toLowerCase();
      
            tags = [start_tag, end_tag]
    }

        is_bold_nodes.push(containsOnly(['strong', 'b'], tags));
    }
    
    return (! is_bold_nodes.includes(false))
}

function getSelectionAsHtml() {
    var html = "";
    if (typeof window.getSelection != "undefined") {
        var sel = window.getSelection();
        if (sel.rangeCount) {
            var container = document.createElement("div");
            for (var i = 0, len = sel.rangeCount; i < len; ++i) {
                container.appendChild(sel.getRangeAt(i).cloneContents());
            }
            html = container.innerHTML;
        }
    } else if (typeof document.selection != "undefined") {
        if (document.selection.type == "Text") {
            html = document.selection.createRange().htmlText;
        }
    }
    return html;
}

function containsOnly(array1, array2){
  return !array2.some(elem => !array1.includes(elem))
}