ScrollController.jumpTo() returns to starting position after ending the gesture with GestureDetectors pan

Question

I am trying to get a Data Table that is too big for the screen to be scrollable on windows platform. Here is the main code:

import '../../imports.dart';

class TablicaTermina extends StatefulWidget {
  final List frizeri;
  const TablicaTermina({super.key, required this.frizeri});

  @override
  State<TablicaTermina> createState() => _TablicaTerminaState();
}

class _TablicaTerminaState extends State<TablicaTermina> {
  // Scroll
  ScrollController horizontalController = ScrollController();
  ScrollController verticalController = ScrollController();
  double totalDeltaX = 0.0;
  double totalDeltaY = 0.0;

  @override
  Widget build(BuildContext context) {
    return Padding(
      padding: const EdgeInsets.only(left: 15, right: 15, bottom: 10),
      child: Column(
        children: [
          Row(children: [
            DataTable(
                border: TableBorder(
                    horizontalInside:
                        BorderSide(color: Colors.red.withOpacity(0.5))),
                columns: const [
                  DataColumn(
                      label: Text("data",
                          style: TextStyle(color: Colors.transparent)))
                ],
                rows: const []),
            DataTable(
                columns: List<DataColumn>.generate(
                    widget.frizeri.length,
                    (index) => DataColumn(
                            label: Text(
                          widget.frizeri[0][1],
                          style: const TextStyle(color: Colors.white),
                        ))),
                rows: const []),
          ]),
          SingleChildScrollView(
            scrollDirection: Axis.horizontal,
            child: Row(
              children: [
                DataTable(
                  headingRowHeight: 0,
                  columns: const [DataColumn(label: Text(""))],
                  rows: List<DataRow>.generate(
                      widget.frizeri.length * 3,
                      (index) => const DataRow(cells: [
                            DataCell(Text(
                              "Nig,",
                              style: TextStyle(color: Colors.amberAccent),
                            ))
                          ])),
                ),
                GestureDetector(
                  onPanUpdate: (details) {
                    handleMouseMoveScroll(details);
                  },
                  child: SingleChildScrollView(
                    scrollDirection: Axis.vertical,
                    controller: verticalController,
                    child: SingleChildScrollView(
                      scrollDirection: Axis.horizontal,
                      controller: horizontalController,
                      child: DataTable(
                        headingRowHeight: 0,
                        columns: List<DataColumn>.generate(
                            widget.frizeri.length * 3,
                            (index) => DataColumn(
                                    label: Text(
                                  widget.frizeri[0][1],
                                  style: const TextStyle(
                                      color: Colors.transparent),
                                ))),
                        rows: List<DataRow>.generate(
                          widget.frizeri.length * 3,
                          (int rowIndex) => DataRow(
                            cells: List<DataCell>.generate(
                              widget.frizeri.length * 3,
                              (int cellIndex) => const DataCell(
                                SizedBox(
                                    child: Text("Bok",
                                        style: TextStyle(
                                          color: Colors.white,
                                        ))),
                              ),
                            ),
                          ),
                        ),
                      ),
                    ),
                  ),
                ),
              ],
            ),
          ),
        ],
      ),
    );
  }

  /*  void handleMouseMoveScroll(DragUpdateDetails details) {
    setState(() {
      totalDeltaX += details.delta.dx;
      totalDeltaY += details.delta.dy;
    });

    /*  double noviHorizontalOffset = totalDeltaX;
    double noviVerticalOffset = totalDeltaY;

    print("NoviHorizontalOffset before clamp: $noviHorizontalOffset");
    print("NoviVerticalOffset before clamp: $noviVerticalOffset"); */

    /*  // Clamp the offsets to prevent scrolling beyond limits
    noviHorizontalOffset = noviHorizontalOffset.clamp(
        0.0, horizontalController.position.maxScrollExtent);
    noviVerticalOffset = noviVerticalOffset.clamp(
        0.0, verticalController.position.maxScrollExtent); */

    /*   print("NoviHorizontalOffset after clamp: $noviHorizontalOffset");
    print("NoviVerticalOffset after clamp: $noviVerticalOffset"); */

    horizontalController.jumpTo(totalDeltaX);
    verticalController.jumpTo(totalDeltaY);
  } */

  void handleMouseMoveScroll(DragUpdateDetails details) {
    setState(() {
      totalDeltaX -= details.delta.dx;
      totalDeltaY -= details.delta.dy;
    });
    print(totalDeltaX);

    horizontalController.jumpTo(totalDeltaX);
  }
}

The question is with second Data Table inside the Second Row(). It is wrapped with two SingleChildScrollViews that have opposite scroll directions. They are wrapped with GestureDetector and it is listening for pan. That enables me to drag the table so I can see all info. Currently it is to big and I only see part of it:

When dragging the table moves but if I stop moving or let go of mouse it will automatically go back to starting position. I save the last position in totalDeltaX/Y vars and when I start dragging again it starts from that position but if I am not moving the cursor it will just show starting position. Here is the video of the problem:

I have tried a lot of combinations of totalIndex and details values but none seem to work. I managed to make dragging start from last position but it still goes back to starting position as seen on GIF.