How to fetch files inside hidden folder using Media Store API on Android 11

Question

I need to fetch data inside WhatsApp folders on External Storage. As i am targeting API Level 30 i am no longer able to access WhatsApp folders on External Storage. I have implemented Storage Access Framework and got Android/media folder Uri and Document File. And using listFiles() i am able to list files but with filter() and sortedByDescending() functions it becomes very slow.

What i have tried?

• Used Cursor loader with Projection and Selection Arguments but it only worked for non hidden folders like WhatsApp Images and WhatsApp Videos

• It returns empty cursor for hidden folder .Statuses

• Tried replacing MediaStore.Video.Media.EXTERNAL_CONTENT_URI with MediaStore.Video.Media.getContentUri(MediaStore.VOLUME_EXTERNAL)

What is required?

• List images and videos from .Statuses folder same as i am listing WhatsApp Images using Media Store in HomeActivity.java

Below is my code

In this activity i get permision to Android/media and set all WhatsApp folders URIs for status fetching and other use, but fetched WhatsApp Images with projection and selection from WhatsApp Images folder

class HomeActivity : AppCompatActivity(), InternetListener, PurchasesUpdatedListener,
    CoroutineScope {
    private val exceptionHandler = CoroutineExceptionHandler { context, exception ->
        Toast.makeText(this, exception.message, Toast.LENGTH_LONG).show()

    }
    private val dataRepository: DataRepository by inject()
    val tinyDB: TinyDB by inject()

    val REQUEST_CODE = 12123

    init {
        newNativeAdSetUp = null
    }

    val sharedViewModel by viewModel<SharedViewModel>()

    val viewModel by viewModel<HomeViewModel>()


    val handler = CoroutineExceptionHandler { _, exception ->
        Log.d("CoroutineException", "$exception handled !")
    }
    override val coroutineContext: CoroutineContext
        get() = Dispatchers.Main + job + handler
    private lateinit var job: Job
    val sdk30PermissionListener = object : PermissionListener {
        override fun onPermissionGranted() {
            openDocumentTree()
        }

        override fun onPermissionDenied(deniedPermissions: MutableList<String>?) {
        }
    }


    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_home)

        handlePermissionsByVersion()
    }

    private fun handlePermissionsByVersion() {

        if (SDK_INT >= Build.VERSION_CODES.R) {
            if ((ContextCompat.checkSelfPermission(
                    this,
                    Manifest.permission.WRITE_EXTERNAL_STORAGE
                )
                        == PackageManager.PERMISSION_GRANTED) && (ContextCompat.checkSelfPermission(
                    this,
                    Manifest.permission.READ_EXTERNAL_STORAGE
                )
                        == PackageManager.PERMISSION_GRANTED)
            ) {
                //if granted load whatsapp images and some uris setup to viewmodel
                loadWhatsAppImages()
                if (arePermissionsGranted()) {
                    if (dataRepository.mrWhatsAppImages == null || dataRepository.mrWhatsAppBusinessImages == null) {
                        setUpWAURIs()
                    }
                }
            } else {
                TedPermission.with(this)
                    .setPermissionListener(sdk30PermissionListener)
                    .setDeniedMessage("If you reject permission,you can not use this service\n\nPlease turn on permissions at [Setting] > [Permission]")
                    .setPermissions(
                        Manifest.permission.WRITE_EXTERNAL_STORAGE,
                        Manifest.permission.READ_EXTERNAL_STORAGE
                    )
                    .check()
            }

        }
    }

    override fun onActivityResult(requestCode: Int, resultCode: Int, @Nullable data: Intent?) {
        super.onActivityResult(requestCode, resultCode, data)


        if (resultCode == RESULT_OK && requestCode == REQUEST_CODE) {
            if (data != null) {
                //this is the uri user has provided us
                val treeUri: Uri? = data.data
                if (treeUri != null) {
                    sharedViewModel.treeUri = treeUri
                    val decoded = Uri.decode(treeUri.toString())
                    Log.i(LOGTAG, "got uri: ${treeUri.toString()}")
                    // here we should do some checks on the uri, we do not want root uri
                    // because it will not work on Android 11, or perhaps we have some specific
                    // folder name that we want, etc
                    if (Uri.decode(treeUri.toString()).endsWith(":")) {
                        showWrongFolderSelection()
                        return
                    }
                    if (!decoded.equals(Constants.WHATSAPP_MEDIA_URI_DECODED)) {
                        showWrongFolderSelection()
                        return
                    }
                    // here we ask the content resolver to persist the permission for us
                    val takeFlags: Int = Intent.FLAG_GRANT_READ_URI_PERMISSION or
                            Intent.FLAG_GRANT_WRITE_URI_PERMISSION
                    contentResolver.takePersistableUriPermission(
                        treeUri,
                        takeFlags
                    )
                    val treeUriAsString = treeUri.toString()
                    tinyDB.putString("FOLDER_URI", treeUriAsString)
                    if (SDK_INT >= Build.VERSION_CODES.R) {
                        setupPaths()
                    }

                }
            }
        }
    }

    private fun setupPaths() {
        setUpOverlay()
        fetchWhatsAppRootURIs(
            this,
            sharedViewModel,
            dataRepository,
            tinyDB
        ) {
            fetchWhatsAppBusinessRootURIs(
                this,
                sharedViewModel,
                dataRepository,
                tinyDB
            ) {
                tinyDB.putBoolean("WARootPathsDone", true)
                removeOverlay()
            }
        }


    }

    override fun onDestroy() {
        dialogHandler.removeCallbacksAndMessages(null)
        super.onDestroy()
    }
    val loadmanagerImages = object : LoaderManager.LoaderCallbacks<Cursor> {
        val whatsAppImagesArrayList = arrayListOf<File>()


        override fun onCreateLoader(id: Int, args: Bundle?): Loader<Cursor> {
            var location: File = File(
                Environment.getExternalStorageDirectory()
                    .toString() + Constants.whatsapp_images_path
            )
            if (!location.exists()) {
                location = File(
                    Environment.getExternalStorageDirectory()
                        .toString() + Constants.whatsapp_images_path11
                )
            }

            if (location != null && location.exists()) {
                whatsAppImagesArrayList.clear()
                Timber.e("checkLoaded-onCreateLoader $id")
                if (id == 0) {
                    var folder = location.absolutePath
                    val projection = arrayOf(
                        MediaStore.MediaColumns.DATA,
                        MediaStore.MediaColumns.DATE_MODIFIED
                    )
                    val selection = MediaStore.Images.Media.DATA + " like ? "
                    val selectionArgs: String = "%$folder%"

                    return CursorLoader(
                        this@HomeActivity,
                        MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
                        projection,
                        selection,
                        arrayOf(selectionArgs),
                        "${MediaStore.Images.Media.DATE_MODIFIED} DESC"
                    )
                }
            }

            return null!!
        }

        override fun onLoadFinished(loader: Loader<Cursor>, cursor: Cursor?) {
            Timber.e("checkLoaded-onLoadFinished")
            var absolutePathOfImage: String
            if (loader.id == 0) {
                cursor?.let {
                    val columnIndexData = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DATA)
                    GlobalScope.launch(Dispatchers.Main + exceptionHandler) {

                        async(Dispatchers.IO + exceptionHandler) {
                            while (!cursor.isClosed && cursor.moveToNext() == true) {
                                absolutePathOfImage = cursor.getString(columnIndexData!!)
                                whatsAppImagesArrayList.add(File(absolutePathOfImage))

                            }
                        }.await()
                        LoaderManager.getInstance(this@HomeActivity).destroyLoader(0)
                        Timber.e("checkLoaded-Completion")
                        galleryViewModel.whatsAppImagesList.postValue(whatsAppImagesArrayList)
                    }


                }
            }
        }

        override fun onLoaderReset(loader: Loader<Cursor>) {
        }

    }

    fun loadWhatsAppImages() {
        try {
            tinyDB.putBoolean("whatsAppMediaLoadCalled", true)
            LoaderManager.getInstance(this).initLoader(
                0,
                null,
                loadmanagerImages
            )
        } catch (e: RuntimeException) {
            Log.e("exVideos ", "ex : ${e.localizedMessage}")
        }

    }


    companion object {
        const val ANDROID_DOCID = "primary:Android/media/"
        const val EXTERNAL_STORAGE_PROVIDER_AUTHORITY = "com.android.externalstorage.documents"
        private val androidUri = DocumentsContract.buildDocumentUri(
            EXTERNAL_STORAGE_PROVIDER_AUTHORITY, ANDROID_DOCID
        )
        val androidTreeUri = DocumentsContract.buildTreeDocumentUri(
            EXTERNAL_STORAGE_PROVIDER_AUTHORITY, ANDROID_DOCID
        )
    }

    private fun openDocumentTree() {
        val uriString = tinyDB.getString("FOLDER_URI", "")
        when {
            uriString == "" -> {
                Log.w(LOGTAG, "uri not stored")
                askPermission()
            }
            arePermissionsGranted() -> {
            }
            else -> {
                Log.w(LOGTAG, "uri permission not stored")
                askPermission()
            }
        }
    }
    // this will present the user with folder browser to select a folder for our data
    private fun askPermission() {
        val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE)
        intent.putExtra(DocumentsContract.EXTRA_INITIAL_URI, androidUri)
        startActivityForResult(intent, REQUEST_CODE)
    }

    private fun arePermissionsGranted(): Boolean {
        var uriString = tinyDB.getString("FOLDER_URI", "")
        val list = contentResolver.persistedUriPermissions
        for (i in list.indices) {
            val persistedUriString = list[i].uri.toString()
            if (persistedUriString == uriString && list[i].isWritePermission && list[i].isReadPermission) {
                return true
            }
        }
        return false
    }

    private fun showWrongFolderSelection() {
        val layoutInflaterAndroid = LayoutInflater.from(this)
        val mView = layoutInflaterAndroid.inflate(R.layout.layout_dialog_wrong_folder, null)
        val builder = AlertDialog.Builder(this, R.style.ThemePageSearchDialog)
        builder.setView(mView)
        val alertDialog = builder.show()
        alertDialog.setCancelable(false)
        val btnOk = mView.findViewById(R.id.tvExit) as TextView
        val tvCancel = mView.findViewById(R.id.tvCancel) as TextView
        btnOk.setOnClickListener {
            alertDialog.dismiss()
            openDocumentTree()
        }
        tvCancel.setOnClickListener {
            alertDialog.dismiss()
        }

    }

    private fun setUpWAURIs() {
        dataRepository.mrWhatsAppImages =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppImages")
            )
        dataRepository.mrWhatsAppVN =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppVN")
            )
        dataRepository.mrWhatsAppDocs =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppDocs")
            )
        dataRepository.mrWhatsAppVideo =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppVideo")
            )
        dataRepository.mrWhatsAppAudio =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppAudio")
            )
        dataRepository.WhatsAppStatuses =
            getDocumentFileFromStringURIStatuses(
                this,
                tinyDB.getString("WhatsAppStatuses")
            )



        dataRepository.mrWhatsAppBusinessImages =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppBusinessImages")
            )
        dataRepository.mrWhatsAppBusinessVN =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppBusinessVN")
            )
        dataRepository.mrWhatsAppBusinessDocs =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppBusinessDocs")
            )
        dataRepository.mrWhatsAppBusinessVideo =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppBusinessVideo")
            )
        dataRepository.mrWhatsAppBusinessAudio =
            getDocumentFileFromStringURI(
                this,
                tinyDB.getString("mrWhatsAppBusinessAudio")
            )
        dataRepository.WhatsAppBusinessStatuses =
            getDocumentFileFromStringURIStatuses(
                this,
                tinyDB.getString("WhatsAppBusinessStatuses")
            )
    }

    fun setUpOverlay() {
        val dialogfragment = FullScreenLoadingDialog()
        dialogfragment.isCancelable = false
        dialogfragment.setisAdmobAd(true)
        val ft: FragmentTransaction =
            supportFragmentManager.beginTransaction()
        ft.add(dialogfragment, "DialogFragment_FLAG")
        ft.commitAllowingStateLoss()
    }

    fun removeOverlay() {
        val fragment: Fragment? = supportFragmentManager.findFragmentByTag("DialogFragment_FLAG")
        if (fragment != null && fragment is DialogFragment) {
            fragment.dismissAllowingStateLoss()
        }
    }
    fun fetchWhatsAppRootURIs(
        context: Context,
        sharedViewModel: SharedViewModel,
        dataRepository: DataRepository,
        tinyDB: TinyDB, completed: () -> Unit

    ) {
        val selectedPackageName = Constants.WHATSAPP_PKG_NAME
        val selectedRootName = Constants.WHATSAPP_ROOT_NAME
        var waImages: DocumentFile? = null
        var waVN: DocumentFile? = null
        var waDocs: DocumentFile? = null
        var waVideos: DocumentFile? = null
        var waAudio: DocumentFile? = null
        var waStatus: DocumentFile? = null
        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && sharedViewModel.treeUri != null) {
            CoroutineScope(Dispatchers.Main).launch {
                async(Dispatchers.IO) {
                    val dir = DocumentFile.fromTreeUri(
                        context,
                        sharedViewModel.treeUri!!
                    )
                    dir?.listFiles()?.forEach {
                        if (it.name.equals(selectedPackageName)) {
                            it.listFiles().forEach {
                                if (it.name.equals(selectedRootName)) {
                                    it.listFiles().forEach {
                                        if (it.name.equals(Constants.WHATSAPP_MEDIA_FOLDER_NAME)) {
                                            it.listFiles().forEach {
                                                if (it.name.equals(Constants.FOLDER_NAME_WHATSAPP_IMAGES)) {
                                                    waImages = it

                                                } else if (it.name.equals(Constants.FOLDER_NAME_WHATSAPP_VN)) {
                                                    waVN = it

                                                } else if (it.name.equals(Constants.FOLDER_NAME_WHATSAPP_DOCUMENTS)) {
                                                    waDocs = it
                                                } else if (it.name.equals(Constants.FOLDER_NAME_WHATSAPP_VIDEO)) {
                                                    waVideos = it

                                                } else if (it.name.equals(Constants.FOLDER_NAME_WHATSAPP_AUDIO)) {
                                                    waAudio = it

                                                } else if (it.name.equals(Constants.FOLDER_NAME_STATUSES)) {
                                                    waStatus = it

                                                }

                                            }


                                        }
                                    }
                                }
                            }
                        }


                    }
                }.await()
                Timber.e("processStatusFetch:Done")
                tinyDB.putString("mrWhatsAppImages", waImages?.uri.toString())
                tinyDB.putString("mrWhatsAppVN", waImages?.uri.toString())
                tinyDB.putString("mrWhatsAppDocs", waImages?.uri.toString())
                tinyDB.putString("mrWhatsAppVideo", waImages?.uri.toString())
                tinyDB.putString("mrWhatsAppAudio", waImages?.uri.toString())
                tinyDB.putString("WhatsAppStatuses", waStatus?.uri.toString())

                dataRepository.mrWhatsAppImages = waImages
                dataRepository.mrWhatsAppVN = waVN
                dataRepository.mrWhatsAppDocs = waDocs
                dataRepository.mrWhatsAppVideo = waVideos
                dataRepository.mrWhatsAppAudio = waAudio
                dataRepository.WhatsAppStatuses = waStatus
                completed()


            }
        }
    }

Here i am using .Statuses folder URI to list DocumentFiles and display but this way it is slow

class StatusImageFragment : Fragment(), StatusListener, CoroutineScope {

    companion object {
        fun newInstance() = StatusImageFragment()
    }

    val handler = CoroutineExceptionHandler { _, exception ->
        Log.d("CoroutineException", "$exception handled !")
    }
    override val coroutineContext: CoroutineContext
        get() = Dispatchers.Main + job + handler
    private lateinit var job: Job

    private var adapterSDK30 = StatusImageAdapterSDK30()
    private var no_image: ImageView? = null
    private var no_image_txt: TextView? = null

    val tinyDB: TinyDB by inject()
    val sharedViewModel by viewModel<SharedViewModel>()
    private val dataRepository: DataRepository by inject()

    override fun onCreateView(
        inflater: LayoutInflater, container: ViewGroup?,
        savedInstanceState: Bundle?
    ): View? {
        job = Job()
        return inflater.inflate(R.layout.status_image_fragment, container, false)
    }

    override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
        super.onViewCreated(view, savedInstanceState)

        swipeRefresh(false, false)
    }


    public fun swipeRefresh(isReloadRequired: Boolean, isFromModeChanged: Boolean) {
        try {
            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
                if (isFromModeChanged) {
                    status_image_recycler.visibility = View.GONE
                    progressbar.visibility = View.VISIBLE
                    no_image?.let {
                        it.visibility = View.GONE
                    }
                    no_image_txt?.let {
                        it.visibility = View.GONE
                    }
                    go_to_app?.let {
                        it.visibility = View.GONE
                    }
                } else {
                    if (adapterSDK30.listImages == null || adapterSDK30.listImages.size == 0) {
                        no_image?.let {
                            it.visibility = View.GONE
                        }
                        no_image_txt?.let {
                            it.visibility = View.GONE
                        }
                        go_to_app?.let {
                            it.visibility = View.GONE
                        }
                        progressbar.visibility = View.VISIBLE
                    }
                }
                if (isReloadRequired) {
                    processStatusFetchFromChild({
                        sharedViewModel.statusImages.observe(viewLifecycleOwner, Observer {
                            val arrayList = it
                            adapterSDK30.listImages = arrayList
                            postFetchingExecutionSDK30()
                        })
                    })

                } else {
                    sharedViewModel.statusImages.observe(viewLifecycleOwner, Observer {
                        val arrayList = it
                        adapterSDK30.listImages = arrayList
                        adapterSDK30.listImages = it
                        postFetchingExecutionSDK30()
                    })
                }

            }
        } catch (ex: Exception) {
            ex.printStackTrace()

        }

    }

    private fun postFetchingExecutionSDK30() {
        progressbar.visibility = View.GONE
        status_image_recycler.visibility = View.VISIBLE
        if (adapterSDK30!!.listImages != null && adapterSDK30!!.listImages.size > 0) {
            no_image?.let {
                it.visibility = View.GONE
            }
            no_image_txt?.let {
                it.visibility = View.GONE
            }
            go_to_app?.let {
                it.visibility = View.GONE
            }
        } else {
            no_image?.let {
                it.visibility = View.VISIBLE
            }
            no_image_txt?.let {
                it.visibility = View.VISIBLE
            }
            go_to_app?.let {
                it.visibility = View.VISIBLE
            }
        }
        adapterSDK30!!.notifyDataSetChanged()

        status_img_swipe.isRefreshing = false
    }


    override fun onDestroyView() {
        job.cancel()
        super.onDestroyView()
    }


    fun processStatusFetchFromChild(completed: () -> Unit) {
        val statusSelection = tinyDB.getInt(Constants.status_accounts)
        if (statusSelection == 0 || statusSelection == 1) {
            if (dataRepository.WhatsAppStatuses == null) {
                (activity as StatusActivity).setUpWAURIs()
            }
            var documentFileStatuses: DocumentFile? = dataRepository.WhatsAppStatuses
            if (statusSelection == 1) {
                documentFileStatuses = dataRepository.WhatsAppBusinessStatuses
            }
            if (documentFileStatuses != null) {
                launch(Dispatchers.Main) {
                    val statusImages1 = arrayListOf<DocumentFile>()

                    async(Dispatchers.IO) {
                        //this takes time ; want to fetch this same as WhatsApp Gallery
                        statusImages1.addAll(documentFileStatuses!!.listFiles().filter {
                            it.mimeType.equals(Constants.MIME_TYPE_IMG_PNG) || it.mimeType.equals(
                                Constants.MIME_TYPE_IMG_JPG
                            ) || it.mimeType.equals(Constants.MIME_TYPE_IMG_JPEG)
                        }.sortedByDescending { it.lastModified() })
                    }.await()
                    Timber.e("processStatusFetch:Done")
                    sharedViewModel.statusImages.postValue(statusImages1)
                    completed()
                }
            } else {
                Timber.e("processStatusFetch:Done")
                sharedViewModel.statusImages.postValue(arrayListOf<DocumentFile>())
                completed()
            }
        } else {
            Timber.e("processStatusFetch:Done")
            sharedViewModel.statusImages.postValue(arrayListOf<DocumentFile>())
            completed()


        }
    }

}

Please note WhatsApp folder path which i used is

val whatsapp_images_path11 = "/Android/media/“ +"com.whatsapp" +"/WhatsApp/Media/WhatsAppImages/"

How i can use MediaStore in this case so that i don't need to use sort and filter functions of list? Its not important to get java.io File only i can work with URIs as well.

Answer 1

Using DocumentFile to handle SAF uries is slow indeed.

Better use DocumentsContract to do so.

Its about twenty times as fast as DocumentFile and about as fast as classic File class stuff.

Using MediaStore for hidden folders should be possible. You cannot create hidden folders with the mediastore. But if you managed to make them not using mediastore you should be able to list files in them using mediastore. Well if they are scanned. And if they belong to your app.

Answer 2

What I have finally implemented, in android 10+ you need to ask the user for your specific directory access. Then you can use this functions to fetch statuses:

@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
private fun readSDKFrom30(): ArrayList<String> {
    val treeUri = DocumentsContract.buildTreeDocumentUri(
        EXTERNAL_STORAGE_PROVIDER_AUTHORITY,
        "primary:Android/media/com.whatsapp/WhatsApp/Media/.Statuses"
    )
    val tree = DocumentFile.fromTreeUri(context, treeUri)!!
    val pathList = ArrayList<String>()

    listFolderContent(tree).forEach { uri ->
        val file = createFileFromContentUri(uri)
        pathList.add(file.toString())
    }
    return pathList
}

private fun listFolderContent(folder: DocumentFile): List<Uri> {
    return if (folder.isDirectory) {
        val files = folder.listFiles().toMutableList()
        files.sortByDescending { it.lastModified() }

        files.mapNotNull { file ->
            if (file.name != null) file.uri else null
        }
    } else {
        emptyList()
    }
}

@RequiresApi(Build.VERSION_CODES.O)
private fun createFileFromContentUri(fileUri: Uri): File {

    var fileName = ""

    fileUri.let { returnUri ->
        context.contentResolver.query(returnUri, null, null, null)
    }?.use { cursor ->
        val nameIndex = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME)
        cursor.moveToFirst()
        fileName = cursor.getString(nameIndex)
    }

    val iStream: InputStream =
        context.contentResolver.openInputStream(fileUri)!!
    val outputDir: File = context.cacheDir!!
    val outputFile = File(outputDir, fileName)
    copyStreamToFile(iStream, outputFile)
    iStream.close()
    return outputFile
}

private fun copyStreamToFile(inputStream: InputStream, outputFile: File) {
    inputStream.use { input ->
        val outputStream = FileOutputStream(outputFile)
        outputStream.use { output ->
            val buffer = ByteArray(4 * 1024) // buffer size
            while (true) {
                val byteCount = input.read(buffer)
                if (byteCount < 0) break
                output.write(buffer, 0, byteCount)
            }
            output.flush()
        }
    }
}