Hand recognition with matlab

Question

I am trying to on going detect the edge of my fist from images, I am using Matlab environment. I don't know the lightness and colors of the image so I first photo the backround without the hand and for the next images I will reduse the backround photo from it. Next I convert the image from RGB to HSV, LAB and gray scale. I convert train image of only the crop fist from the image to vetors and take the hue vector from hsv and a,b from lab and a vector of the gray scale image and insert it to a GMM in order to receive masking image of only the fist in white and the rest black. here is an example of the pictures I get

HSV and LAB convert:

lab = rgb2lab(img);

a = reshape(lab(:,:,2), [], 1);
b = reshape(lab(:,:,3), [], 1);

hsv = rgb2hsv(img);
h = reshape(hsv(:,:,1), [], 1);
gray = reshape(rgb2gray(img), [], 1);

mat1 = [h a b gray];

Then I use detectBRISKFeatures function to detect the features of the croped image of the fist and compare it with the next features of the images I will get in order to locate only the fist and not anything else (I use matchFeatures for it).

My solution is working but the location that I get is not stable even if the image is the same. How can I have better fist recognition, or what am I doing wrong?

Here is the code I wrote, I get fine hand recognition but the algorithm is slow and not stable:

cam = webcam('Microsoft® LifeCam HD-3000');
cam.WhiteBalanceMode = 'auto';

pause(0.1);

cam.WhiteBalanceMode = 'auto';
cam.ExposureMode = 'auto';

for i = 1:40
    ref = double(snapshot(cam))/255;
end

'screen shot - keep your hand in square untill detection start'
pause(5);

area = [150 100 300 200];

for i = 1:50
    img = double(snapshot(cam))/255;
    shape_img = insertShape(img, "rectangle", area, LineWidth = 10); % inserts circle aroud the cap
    figure(2); imshow(shape_img);
end

for i = 1:30
    img = double(snapshot(cam))/255;
    shape_img = insertShape(img, "rectangle", area, LineWidth = 10); % inserts circle aroud the cap
    imshow(shape_img);
    img = imcrop(img-ref, area);
    lab = rgb2lab(img);
    a = reshape(lab(:,:,2), [], 1);
    b = reshape(lab(:,:,3), [], 1);
    hsv = rgb2hsv(img);
    h = reshape(hsv(:,:,1), [], 1);
    gray = reshape(rgb2gray(img), [], 1);
    mat1 = [h a b gray];
    if i == 1
        mat = [mat1];
    else
        mat = [mat; mat1];
    end
end

'GMM analyze'

tic
GMModel = fitgmdist(mat, 2, 'Replicates', 2, 'CovarianceType','diagonal','SharedCovariance',true);

toc

test_img = double(snapshot(cam))/255;
test_img = imcrop(test_img-ref, area);
lab = rgb2lab(test_img);
a = reshape(lab(:,:,2), [], 1);
b = reshape(lab(:,:,3), [], 1);
hsv = rgb2hsv(test_img);
h = reshape(hsv(:,:,1), [], 1);
gray = reshape(rgb2gray(test_img), [], 1);
test_hsv = [h a b gray];
indx = cluster(GMModel, test_hsv);

N = size(hsv,1);
M = size(hsv,2);

mask1 = reshape(indx(1:M*N), N, M);

seg11 = (mask1 - 1);
se = strel('disk', 4);
BinaryMask = seg11;
BinaryMask = imdilate(imerode(seg11, se), se);
BinaryMask = imbilatfilt(BinaryMask);
BinaryMask = bwareafilt(BinaryMask > 0.7, 1);
curr_points = detectBRISKFeatures(BinaryMask); 
[curr_features, curr_valid_points] = extractHOGFeatures(BinaryMask, curr_points);
'keep still untill here!!!!!'`
 
hand recognition 
pos = [200 200];
rect = area;
count = 0;
while(true)
    count = count+1;
    hand = double(snapshot(cam))/255;
    a = rgb2gray(hand);
    if mod(count,2) == 0
        [pos, rect] = mask_hand2(hand , GMModel, pos, curr_features, curr_valid_points, rect, BinaryMask, ref);
        last_hand = rgb2gray(hand);
    end
end


function [pos_area rect] = mask_hand2(test_img, GMModel, last_pos, 
    new_img = test_img;
    test_img = imcrop(test_img-ref, rect);
    lab = rgb2lab(test_img);
    a = reshape(lab(:,:,2), [], 1);
    b = reshape(lab(:,:,3), [], 1);
    hsv = rgb2hsv(test_img);
    h = reshape(hsv(:,:,1), [], 1);
    gray = reshape(rgb2gray(test_img), [], 1);
    test_mat = [h a b gray];
    indx = cluster(GMModel, test_mat);
    N = size(hsv,1);
    M = size(hsv,2);
    mask1 = reshape(indx(1:M*N), N, M);
    seg11 = (mask1 - 1);
    se = strel('disk', 2);
    BinaryMask = imdilate(imerode(seg11, se), se);
    BinaryMask = bwareafilt(BinaryMask > 0.7, 1); % Extract largest blob.
    points = detectBRISKFeatures(BinaryMask); 
    [features, valid_points] = extractHOGFeatures(BinaryMask, points);
    index_pairs = matchFeatures(features, curr_features);
    if isempty(index_pairs)
        rect = [0 0 size(new_img, 2) size(new_img, 1)];
        pos_area = last_pos;
    else
        matchedPtsOriginal = curr_valid_points(index_pairs(:,2));
        matchedPtsTarget = valid_points(index_pairs(:,1));
        try
            if size(find(matchedPtsTarget.Location == last_pos),1) ~= 2 
                pos_area = min(matchedPtsTarget.Location) + [rect(1) rect(2)];
            end
        end
        last_pos = pos_area;

        try
            rect = rectangle_position(pos_area);
        end
    end
end

function [rect] = rectangle_position(point)
    if point(1)-150 < 0
        x = 0;
    else
        x = point(1)-150;
    end
    if point(2)-100 < 0
        y = 0;
    else
        y = point(2)-100;
    end
    if x > 340
        x = 340;
    end
    if y > 280
        y = 280;
    end
    rect = [x y 300 200]; % [x y width height]
end