RecyclerView inside of ViewPager2 Fragment will not stay scrolled to the bottom

Question

I have a vertical RecyclerView with ListAdapter inside a horizontal ViewPager2. My list items have Chronometers and sections that will show/hide on click. When I scroll to the bottom of the list things jumps back up to around the 3rd to last item in the list. Fragment and layout with ViewPager:

class WorkersFragment : Fragment() {
    private lateinit var viewPager: ViewPager2

    override fun onCreateView(
        inflater: LayoutInflater, container: ViewGroup?,
        savedInstanceState: Bundle?
    ): View? {
        // Inflate the layout for this fragment
        val root = inflater.inflate(R.layout.fragment_workers, container, false)
        // Instantiate a ViewPager2 and a PagerAdapter.
        viewPager = root.findViewById(R.id.workers_ViewPager)

        // The pager adapter, which provides the pages to the view pager widget.
        val pagerAdapter = activity?.let { ScreenSlidePagerAdapter(it) }
        viewPager.adapter = pagerAdapter
        val tabLayout = root.findViewById<TabLayout>(R.id.workers_tab_layout)
        TabLayoutMediator(tabLayout, viewPager) { tab, position ->
            tab.text = resources.getStringArray(R.array.worker_choices)[position]
        }.attach()
        return root
    }

    private inner class ScreenSlidePagerAdapter(fa: FragmentActivity) : FragmentStateAdapter(fa) {
        override fun getItemCount(): Int = 2

        override fun createFragment(position: Int): Fragment {
            return if (position == 0) {
                EmployeesFragment()
            } else {
                CrewsFragment()
            }
        }
    }
}

<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
 xmlns:tools="http://schemas.android.com/tools"
 android:layout_width="match_parent"
 android:layout_height="match_parent"
 xmlns:app="http://schemas.android.com/apk/res-auto"
 android:orientation="vertical"
 tools:context=".ui.workers.WorkersFragment">

 <com.google.android.material.tabs.TabLayout
     android:id="@+id/workers_tab_layout"
     android:layout_width="match_parent"
     android:layout_height="wrap_content"
     android:orientation="horizontal"
     app:layout_constraintTop_toTopOf="parent"
     app:layout_constraintStart_toStartOf="parent"
     app:layout_constraintEnd_toEndOf="parent"/>

 <androidx.viewpager2.widget.ViewPager2
     android:id="@+id/workers_ViewPager"
     android:layout_width="match_parent"
     android:layout_height="0dp"
     android:orientation="horizontal"
     app:layout_constraintTop_toBottomOf="@id/workers_tab_layout"
     app:layout_constraintBottom_toBottomOf="parent"
     app:layout_constraintStart_toStartOf="parent"
     app:layout_constraintEnd_toEndOf="parent"
/>
</androidx.constraintlayout.widget.ConstraintLayout>

Fragment and layout with RecyclerView

class EmployeesFragment : Fragment() {

    private lateinit var viewModel: EmployeesViewModel
    private lateinit var employeeListAdapter: EmployeeListAdapter
    override fun onCreateView(
        inflater: LayoutInflater, container: ViewGroup?,
        savedInstanceState: Bundle?
    ): View? {
        val root = inflater.inflate(R.layout.employees_fragment, container, false)
        val recyclerView = root.findViewById<RecyclerView>(R.id.employees_recyclerview)
        employeeListAdapter = EmployeeListAdapter(object : EmployeeListAdapter.OnEmployeeClickListener {
            override fun onEmployeeClick(employee: EmployeesViewModel.EmployeeDisplayData) {
                viewModel.updateWork(employee)
            }
        })
        recyclerView.adapter = employeeListAdapter
        recyclerView.layoutManager = LinearLayoutManager(this.context)
        viewModel = ViewModelProvider(this).get(EmployeesViewModel::class.java)
        viewModel.employeeDisplayList.observe(
            viewLifecycleOwner
        ) { data ->
            data?.let {
                employeeListAdapter.addDividersAndSubmitList(it)
            }
        }
        val fab = root.findViewById<FloatingActionButton>(R.id.new_employee_fab)
        fab.setOnClickListener {
            val intent = Intent(context, NewEmployeeActivity::class.java)
            startActivityForResult(intent, 456)
        }
        return root.rootView
    }
}

<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:nestedScrollingEnabled="false"
    xmlns:tools="http://schemas.android.com/tools">


    <androidx.recyclerview.widget.RecyclerView
        android:id="@+id/employees_recyclerview"
        android:layout_height="match_parent"
        android:layout_width="0dp"
        android:padding="6dp"
        android:orientation="vertical"
        tools:listitem="@layout/employee_card_item"
        app:layout_constraintLeft_toLeftOf="parent"
        app:layout_constraintRight_toRightOf="parent"
        app:layout_constraintTop_toTopOf="parent"
        app:layout_constraintBottom_toBottomOf="parent"
        />


    <com.google.android.material.floatingactionbutton.FloatingActionButton
        android:id="@+id/new_employee_fab"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toEndOf="parent"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_marginLeft="16dp"
        android:layout_marginRight="16dp"
        android:layout_marginBottom="16dp"
        android:visibility="gone"
        android:src="@drawable/ic_add_white_24dp"/>

</androidx.constraintlayout.widget.ConstraintLayout>

Adapter and item layout

import android.content.Context
import android.content.Intent
import android.os.SystemClock
import android.util.Log
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.widget.TextView
import androidx.constraintlayout.widget.Group
import androidx.recyclerview.widget.DiffUtil
import androidx.recyclerview.widget.ListAdapter
import androidx.recyclerview.widget.RecyclerView
import com.mow.time.R
import com.mow.time.ui.clients.ClientListAdapter
import com.mow.time.ui.payments.NewPaymentActivity
import com.mow.time.ui.summary.SummaryHolderActivity
import kotlinx.android.synthetic.main.employee_card_item.view.*

/*
Adapter to display all the Employees
 */
class EmployeeListAdapter internal constructor(
    private val workListener: OnEmployeeClickListener
) : ListAdapter<EmployeeListAdapter.EmployeeListItem, RecyclerView.ViewHolder>(EmployeeDiffCallback()) {
    private val DIVIDER_ITEM = 0
    private val REAL_ITEM = 1


    class EmployeeViewHolder(
        val employeeView: View,
        val workListener: OnEmployeeClickListener
    ) : RecyclerView.ViewHolder(employeeView) {
        val subOptions : Group = employeeView.findViewById(R.id.employee_sub_options)
        val root = employeeView
        val employeeAvatar = employeeView.employee_avatar
        var isExpanded = false

        companion object {
            fun from(
                parent: ViewGroup,
                workListener: OnEmployeeClickListener
            ): EmployeeViewHolder {
                val layoutInflater = LayoutInflater.from(parent.context)
                val view = layoutInflater.inflate(R.layout.employee_card_item, parent, false)
                return EmployeeViewHolder(view, workListener)
            }
        }

        fun bind(employeeDisplayData: EmployeeListItem.EmployeeNameListItem) {
            subOptions.visibility = if (isExpanded) View.VISIBLE else View.GONE
//            itemView.isActivated = isExpanded
            employeeView.setOnClickListener {
                isExpanded = !isExpanded
                subOptions.visibility = if (isExpanded) View.VISIBLE else View.GONE
//                itemView.isActivated = isExpanded
            }
            employeeAvatar.text = employeeDisplayData.employeeDisplayData.name[0].toString()
            updateEmployee(employeeDisplayData.employeeDisplayData)
        }


        private fun updateEmployee(employee: EmployeesViewModel.EmployeeDisplayData) {
            employeeView.employee_name.text = employee.name
//            employeeView.startWork.setOnClickListener {
//                workListener.onEmployeeClick(employee)
//            }
//            employeeView.payEmployee.setOnClickListener {
//                val intent = Intent(employeeView.context, NewPaymentActivity::class.java)
//                intent.putExtra("id", employee.id)
//                intent.putExtra("payment_type", "employee")
//                employeeView.payEmployee.context.startActivity(intent)
//            }
//            employeeView.editEmployee.setOnClickListener {
//                val intent = Intent(employeeView.context, NewEmployeeActivity::class.java)
//                intent.putExtra("employee_id", employee.id)
//                employeeView.context.startActivity(intent)
//            }
//            employeeView.workList.setOnClickListener {
//                val intent = Intent(employeeView.context, SummaryHolderActivity::class.java)
//                intent.putExtra("id", employee.id)
//                intent.putExtra("name", employee.name)
//                intent.putExtra("affiliation", "Employee")
//                employeeView.context.startActivity(intent)
//            }

            if (employee.hasWorkRunning){
                employeeView.work_timer.base = (SystemClock.elapsedRealtime() - employee.workDuration)
                employeeView.work_timer.visibility = View.VISIBLE
                if (!employee.isPaused){
                    employeeView.startWork.setImageResource(R.drawable.stop_timer_circle)
                    employeeView.work_timer.start()
                }else{
                    employeeView.startWork.setImageResource(R.drawable.start_timer_circle)
                }
            }else{
                employeeView.work_timer.visibility = View.GONE
                employeeView.startWork.setImageResource(R.drawable.start_timer_circle)
            }
        }
    }

    class TextViewHolder(view: View) : RecyclerView.ViewHolder(view) {
        val letterText: TextView = view.findViewById(R.id.letter)
        fun bind(letter: String) {
            letterText.text = letter
        }

        companion object {
            fun from(parent: ViewGroup): TextViewHolder {
                val layoutInflater = LayoutInflater.from(parent.context)
                val view = layoutInflater.inflate(R.layout.header, parent, false)
                return TextViewHolder(view)
            }
        }
    }

    override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {
        return when (viewType) {
            DIVIDER_ITEM -> TextViewHolder.from(parent)
            REAL_ITEM -> EmployeeViewHolder.from(parent, workListener)
            else -> throw ClassCastException("Unknown viewType ${viewType}")
        }
    }

    override fun getItemViewType(position: Int): Int {
        return when (getItem(position)) {
            is EmployeeListItem.EmployeeNameListItem -> REAL_ITEM
            is EmployeeListItem.DividerItem -> DIVIDER_ITEM
        }
    }


    interface OnEmployeeClickListener {
        fun onEmployeeClick(employee: EmployeesViewModel.EmployeeDisplayData)
    }

    sealed class EmployeeListItem {
        abstract val id: Long

        data class EmployeeNameListItem(val employeeDisplayData: EmployeesViewModel.EmployeeDisplayData) : EmployeeListItem() {
            override val id = employeeDisplayData.id
        }

        data class DividerItem(val letter: String) : EmployeeListItem() {
            override val id = Long.MIN_VALUE

        }
    }

    override fun onBindViewHolder(holder: RecyclerView.ViewHolder, position: Int) {
        when (holder) {
            is EmployeeViewHolder -> {
                val employeeItem = getItem(position) as EmployeeListItem.EmployeeNameListItem
                holder.bind(employeeItem)
            }
            is TextViewHolder -> {
                val letterItem = getItem(position) as EmployeeListItem.DividerItem
                holder.bind(letterItem.letter)
            }
        }
    }

    fun addDividersAndSubmitList(employeeList: List<EmployeesViewModel.EmployeeDisplayData>) {
        val itemList = arrayListOf<EmployeeListItem>()
        val alphabetLetters = arrayListOf<Char>()
        employeeList.forEach {
            if (!alphabetLetters.contains(it.name[0])) {
                alphabetLetters.add(it.name[0])
                itemList.add(EmployeeListItem.DividerItem(it.name[0].toString()))
            }
            itemList.add(EmployeeListItem.EmployeeNameListItem(it))
        }
        submitList(itemList)
    }

    class EmployeeDiffCallback : DiffUtil.ItemCallback<EmployeeListItem>() {
        override fun areItemsTheSame(oldItem: EmployeeListItem, newItem: EmployeeListItem): Boolean = (oldItem.id == newItem.id)
        override fun areContentsTheSame(oldItem: EmployeeListItem, newItem: EmployeeListItem): Boolean = oldItem == newItem
    }
}

<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:layout_margin="10dp"
    app:cardBackgroundColor="@color/cardColor"
    app:cardCornerRadius="10dp">


    <TextView
        android:id="@+id/employee_avatar"
        android:layout_width="35dp"
        android:layout_height="35dp"
        android:background="@drawable/bg_square"
        android:backgroundTint="@color/colorPrimary"
        android:gravity="center"
        android:padding="5dp"
        android:text="B"
        android:textColor="@android:color/white"
        app:layout_constraintStart_toStartOf="parent"
        app:layout_constraintTop_toTopOf="parent" />

    <TextView
        android:id="@+id/employee_name"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_marginStart="15dp"
        android:text="NAME"
        android:textAppearance="@style/TextAppearance.AppCompat.Large"
        android:textColor="@android:color/black"
        app:layout_constraintStart_toEndOf="@id/employee_avatar"
        app:layout_constraintTop_toTopOf="@id/employee_avatar"
        app:layout_constraintBottom_toBottomOf="@id/employee_avatar"/>

    <Chronometer
        android:id="@+id/work_timer"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:textColor="@color/ms_black"
        app:layout_constraintEnd_toEndOf="parent"
        android:visibility="gone"
        app:layout_constraintTop_toTopOf="@id/employee_name"/>

    <androidx.appcompat.widget.AppCompatImageButton
        android:id="@+id/startWork"
        android:layout_width="0dp"
        android:layout_height="match_parent"
        android:src="@drawable/timer_with_square_background"
        android:background="@android:color/transparent"
        app:layout_constraintStart_toStartOf="parent"
        app:layout_constraintTop_toBottomOf="@id/employee_name"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toStartOf="@id/payEmployee"/>

    <androidx.appcompat.widget.AppCompatImageButton
        android:id="@+id/payEmployee"
        android:layout_width="0dp"
        android:layout_height="match_parent"
        android:background="@android:color/transparent"
        android:src="@drawable/payment_with_square_background"
        app:layout_constraintStart_toEndOf="@id/startWork"
        app:layout_constraintTop_toBottomOf="@id/employee_name"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toStartOf="@id/workList" />

    <androidx.appcompat.widget.AppCompatImageButton
        android:id="@+id/workList"
        android:layout_width="0dp"
        android:layout_height="match_parent"
        android:background="@android:color/transparent"
        android:src="@drawable/list_with_square"
        app:layout_constraintStart_toEndOf="@id/payEmployee"
        app:layout_constraintTop_toBottomOf="@id/employee_name"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toStartOf="@id/editEmployee"/>

    <androidx.appcompat.widget.AppCompatImageButton
        android:id="@+id/editEmployee"
        android:layout_width="0dp"
        android:layout_height="match_parent"
        android:background="@android:color/transparent"
        android:src="@drawable/edit_with_square_background"
        app:layout_constraintStart_toEndOf="@id/workList"
        app:layout_constraintTop_toBottomOf="@id/employee_name"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toEndOf="parent" />

    <androidx.constraintlayout.widget.Group
        android:id="@+id/employee_sub_options"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:visibility="gone"
        app:constraint_referenced_ids="editEmployee, workList, payEmployee, startWork"
        />

</androidx.constraintlayout.widget.ConstraintLayout>

I have tried replacing ViewPager2 with ViewPager. I have tried different values for height for a lot of the xml widgets. I have tried using fewer ConstraintLayouts. Nothing seems to work. Changing height will make widgets not behave correctly. Chronometer not updating and view visibility of items not correct. Any ideas would be appreciated.