framer-motion `useInView` does not work when rendered conditonally

Question

I have a pagination component where by default there is 10 items per page. I allowed user to choose infinite scrolling and when this is a case, paginated items are replace by infinite scroll items a new item (button) is added to the dom which uses useInView to trigger next page loading. However, for some reason it does not work. I.e. if I scroll to it, it does not trigger loading (useInView returns false). But when I click it (because it trigger loading on click as well) and more items are loaded, then when I again scroll to it, it properly works. So this first scrolling for ome reason does not work. What might be the reason?

export const InfiniteHomeworks = ({
  pagination,
}: {
  pagination: InputPagination;
}) => {
  const ref = useRef<HTMLButtonElement>(null);
  const queryClient = useQueryClient();
  const isInView = useInView(ref, { margin: '500px' });

  const {
    data: pages,
    isFetchingNextPage,
    hasNextPage,
    fetchNextPage,
    isLoading,
    refetch,
  } = useInfiniteQuery(
    ['ifniniteHomeworks'],
    async ({ pageParam = 0 }) =>
      getHomeworks({ ...pagination, page: pageParam + 1 }),

    {
      getPreviousPageParam: (props) =>
        props.page === 1 ? undefined : props.page || 0 - 1,
      getNextPageParam: (props) =>
        props.hasNextPage ? props.page || 0 + 1 : undefined,
      cacheTime: Infinity,
    }
  );

  useEffect(() => {
    refetch();
  }, [pagination]);

  const data = pages?.pages.map((x) => x.results).flat();

  useEffect(() => {
    if (isInView && !isFetchingNextPage && hasNextPage) fetchNextPage();
    console.log('IsInView', isInView);
  }, [isInView, fetchNextPage]);

  return (
    <>
      <HomeworksList
        data={data}
        isLoading={isLoading}
        total={pages?.pages[0].total || 0}
      />
      <button
        ref={ref}
        onClick={() => fetchNextPage()}
        disabled={!hasNextPage || isFetchingNextPage}
        className='mx-auto my-3'
      >
        {isFetchingNextPage ? (
          <Loader2 className='animate-spin' />
        ) : hasNextPage ? (
          'Więcej'
        ) : (
          <TextGradient className='text-lg font-semibold' color='secondary'>
            KONIEC
          </TextGradient>
        )}
      </button>
    </>
  );
};