perPixelTargetFind not working in nested group. (#5287)

* fix perPixelTargetFind in nested object

* fix: lint

* add methods doc

* doc: update _serachPossibleTarget doc

* test: add test case

* fix: remove pointer check

* fix: object in nested group should normalize just once

* test: add test case

* restore to previous code style

* fix: update test case descriptions

* fix: test case

* test: add skew and angle test case

src/canvas.class.js

@@ -1194,15 +1194,20 @@
     },
 
     /**
+     * Checks point is inside the object.
+     * @param {Object} [pointer] x,y object of point coordinates we want to check.
+     * @param {fabric.Object} obj Object to test against
+     * @param {Object} [globalPointer] x,y object of point coordinates relative to canvas used to search per pixel target.
+     * @return {Boolean} true if point is contained within an area of given object
      * @private
      */
-    _checkTarget: function(pointer, obj) {
+    _checkTarget: function(pointer, obj, globalPointer) {
       if (obj &&
           obj.visible &&
           obj.evented &&
           this.containsPoint(null, obj, pointer)){
         if ((this.perPixelTargetFind || obj.perPixelTargetFind) && !obj.isEditing) {
-          var isTransparent = this.isTargetTransparent(obj, pointer.x, pointer.y);
+          var isTransparent = this.isTargetTransparent(obj, globalPointer.x, globalPointer.y);
           if (!isTransparent) {
             return true;
           }
@@ -1214,20 +1219,26 @@
     },
 
     /**
+     * Function used to search inside objects an object that contains pointer in bounding box or that contains pointerOnCanvas when painted
+     * @param {Array} [objects] objects array to look into
+     * @param {Object} [pointer] x,y object of point coordinates we want to check.
+     * @return {fabric.Object} object that contains pointer
      * @private
      */
     _searchPossibleTargets: function(objects, pointer) {
-
       // Cache all targets where their bounding box contains point.
-      var target, i = objects.length, normalizedPointer, subTarget;
+      var target, i = objects.length, subTarget;
       // Do not check for currently grouped objects, since we check the parent group itself.
       // until we call this function specifically to search inside the activeGroup
       while (i--) {
-        if (this._checkTarget(pointer, objects[i])) {
+        var objToCheck = objects[i];
+        if (this._checkTarget(objToCheck.group && objToCheck.group.type !== 'activeSelection'
+          ? this._normalizePointer(objToCheck.group, pointer)
+          : pointer,
+        objToCheck, pointer)) {
           target = objects[i];
           if (target.subTargetCheck && target instanceof fabric.Group) {
-            normalizedPointer = this._normalizePointer(target, pointer);
-            subTarget = this._searchPossibleTargets(target._objects, normalizedPointer);
+            subTarget = this._searchPossibleTargets(target._objects, pointer);
             subTarget && this.targets.push(subTarget);
           }
           break;

test/unit/canvas.js

@@ -790,6 +790,89 @@
     canvas.remove(triangle);
   });
 
+  QUnit.test('findTarget with perPixelTargetFind in nested group', function(assert) {
+    assert.ok(typeof canvas.findTarget === 'function');
+    var triangle = makeTriangle({ left: 0, top: 0, width: 30, height: 30, fill: 'yellow' }),
+        triangle2 = makeTriangle({ left: 100, top: 120, width: 30, height: 30, angle: 100, fill: 'pink' }),
+        circle = new fabric.Circle({ radius: 30, top: 0, left: 30, fill: 'blue' }),
+        circle2 = new fabric.Circle({ scaleX: 2, scaleY: 2, radius: 10, top: 120, left: -20, fill: 'purple' }),
+        rect = new fabric.Rect({ width: 100, height: 80, top: 50, left: 60, fill: 'green' }),
+        rect2 = new fabric.Rect({ width: 50, height: 30, top: 10, left: 110, fill: 'red', skewX: 40, skewY: 20 }),
+        group1 = new fabric.Group([triangle, circle, rect2], { subTargetCheck: true }),
+        group2 = new fabric.Group([group1, circle2, rect, triangle2], { subTargetCheck: true }),
+        group3 = new fabric.Group([group2], { subTargetCheck: true }),
+        target;
+
+    canvas.add(group3);
+    canvas.perPixelTargetFind = true;
+    target = canvas.findTarget({
+      clientX: 5, clientY: 5
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 1');
+    target = canvas.findTarget({
+      clientX: 21, clientY: 9
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 2');
+    target = canvas.findTarget({
+      clientX: 37, clientY: 7
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 3');
+    target = canvas.findTarget({
+      clientX: 89, clientY: 47
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 4');
+    target = canvas.findTarget({
+      clientX: 16, clientY: 122
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 5');
+    target = canvas.findTarget({
+      clientX: 127, clientY: 37
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 6');
+    target = canvas.findTarget({
+      clientX: 87, clientY: 139
+    });
+    assert.equal(target, null, 'Should return null because of transparency checks case 7');
+    target = canvas.findTarget({
+      clientX: 15, clientY: 15
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 3, 'Subtargets length should be 3');
+    assert.equal(canvas.targets[0], triangle, 'The deepest target should be triangle');
+    target = canvas.findTarget({
+      clientX: 50, clientY: 20
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 3, 'Subtargets length should be 3');
+    assert.equal(canvas.targets[0], circle, 'The deepest target should be circle');
+    target = canvas.findTarget({
+      clientX: 117, clientY: 16
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 3, 'Subtargets length should be 2');
+    assert.equal(canvas.targets[0], rect2, 'The deepest target should be rect2');
+    target = canvas.findTarget({
+      clientX: 100, clientY: 90
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 2, 'Subtargets length should be 2');
+    assert.equal(canvas.targets[0], rect, 'The deepest target should be rect');
+    target = canvas.findTarget({
+      clientX: 9, clientY: 145
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 2, 'Subtargets length should be 2');
+    assert.equal(canvas.targets[0], circle2, 'The deepest target should be circle2');
+    target = canvas.findTarget({
+      clientX: 66, clientY: 143
+    });
+    assert.equal(target, group3, 'Should return the group3 now');
+    assert.equal(canvas.targets.length, 2, 'Subtargets length should be 2');
+    assert.equal(canvas.targets[0], triangle2, 'The deepest target should be triangle2');
+    canvas.perPixelTargetFind = false;
+    canvas.remove(group3);
+  });
+
   QUnit.test('findTarget on activegroup', function(assert) {
     var rect1 = makeRect({ left: 0, top: 0 }), target;
     var rect2 = makeRect({ left: 20, top: 20 });