Add more test for subTargetCheck and zoom + pan (#5347)

* cleanedup and added test

* adding wider test

src/canvas.class.js

@@ -1232,10 +1232,9 @@
       // until we call this function specifically to search inside the activeGroup
       while (i--) {
         var objToCheck = objects[i];
-        if (this._checkTarget(objToCheck.group && objToCheck.group.type !== 'activeSelection'
-          ? this._normalizePointer(objToCheck.group, pointer)
-          : pointer,
-        objToCheck, pointer)) {
+        var pointerToUse = objToCheck.group && objToCheck.group.type !== 'activeSelection' ?
+          this._normalizePointer(objToCheck.group, pointer) : pointer;
+        if (this._checkTarget(pointerToUse, objToCheck, pointer)) {
           target = objects[i];
           if (target.subTargetCheck && target instanceof fabric.Group) {
             subTarget = this._searchPossibleTargets(target._objects, pointer);

test/unit/canvas.js

@@ -101,6 +101,7 @@
       canvas.preserveObjectStacking = fabric.Canvas.prototype.preserveObjectStacking;
     },
     afterEach: function() {
+      canvas.viewportTransform = [1, 0, 0, 1, 0, 0];
       canvas.clear();
       canvas.backgroundColor = fabric.Canvas.prototype.backgroundColor;
       canvas.overlayColor = fabric.Canvas.prototype.overlayColor;
@@ -706,6 +707,85 @@
     canvas.remove(group);
   });
 
+  QUnit.test('findTarget with subTargetCheck and canvas zoom', function(assert) {
+    var rect3 = new fabric.Rect({
+      width: 100,
+      height: 100,
+      fill: 'yellow'
+    });
+    var rect4 = new fabric.Rect({
+      width: 100,
+      height: 100,
+      left: 100,
+      top: 100,
+      fill: 'purple'
+    });
+    var group3 = new fabric.Group(
+      [rect3, rect4],
+      { scaleX: 0.5, scaleY: 0.5, top: 100, left: 0 });
+    group3.subTargetCheck = true;
+
+    var rect1 = new fabric.Rect({
+      width: 100,
+      height: 100,
+      fill: 'red'
+    });
+    var rect2 = new fabric.Rect({
+      width: 100,
+      height: 100,
+      left: 100,
+      top: 100,
+      fill: 'blue'
+    });
+    var g = new fabric.Group([rect1, rect2, group3], { top: -150, left: -50 });
+    g.subTargetCheck = true;
+    canvas.viewportTransform = [0.1, 0, 0, 0.1, 100, 200];
+    canvas.add(g);
+
+    var target = canvas.findTarget({
+      clientX: 96, clientY: 186
+    }, true);
+    assert.equal(target, g, 'Should return the group 96');
+    assert.equal(canvas.targets[0], rect1, 'should find the target rect 96');
+    canvas.targets = [];
+
+    target = canvas.findTarget({
+      clientX: 98, clientY: 188
+    }, true);
+    assert.equal(target, g, 'Should return the group 98');
+    assert.equal(canvas.targets[0], rect1, 'should find the target rect1 98');
+    canvas.targets = [];
+
+    target = canvas.findTarget({
+      clientX: 100, clientY: 190
+    }, true);
+    assert.equal(target, g, 'Should return the group 100');
+    assert.equal(canvas.targets[0], rect1, 'should find the target rect1 100');
+    canvas.targets = [];
+
+    target = canvas.findTarget({
+      clientX: 102, clientY: 192
+    }, true);
+    assert.equal(target, g, 'Should return the group 102');
+    assert.equal(canvas.targets[0], rect1, 'should find the target rect 102');
+    canvas.targets = [];
+
+    target = canvas.findTarget({
+      clientX: 104, clientY: 194
+    }, true);
+    assert.equal(target, g, 'Should return the group 104');
+    assert.equal(canvas.targets[0], rect1, 'should find the target rect 104');
+    canvas.targets = [];
+
+    target = canvas.findTarget({
+      clientX: 106, clientY: 196
+    }, true);
+    assert.equal(target, g, 'Should return the group 106');
+    assert.equal(canvas.targets[0], rect2, 'should find the target rect2 106');
+    canvas.targets = [];
+
+  });
+
   QUnit.test('findTarget with subTargetCheck on activeObject', function(assert) {
     var rect = makeRect({ left: 0, top: 0 }),
         rect2 = makeRect({ left: 30, top:  30}), target,