[messages] Switch from JSON to ZIP / NDJSON format

This is a major rewrite of the message import / export code, that
switches the format from a single (standard) JSON file, with embedded
Base64 encoded MMS binary data, to a ZIP file containing a
Newline-delimited JSON (NDJSON) file ('messasges.ndjson'), containing
message metadata and text data, and a 'data' directory, containing the
untouched binary files stored natively by Android. There are a number of
advantages, as well as some disadvantages, to the new format:

Advantages:
-----------

Separating (encoded) binary data from text data and metadata results in
much cleaner text, which can be much more comfortably browsed by humans.

The ZIP file format is much more flexibile than the monolithic JSON file
format. E.g., additional information about the exporting system and app
and statistics about the export run can be easily included in another
file within the ZIP archive without substantially modifying the existing
export flow (this is not yet implemented, but will likely be in the
future.)

Using ZIP files automatically provides compression, although the
reduction in file size will depend on how much of the exported data is
compressible text (i.e., metadata and text data), as opposed to binary
data, which will generally be already compressed and not able to be
compressed much further.

Not including the binary data in the (ND)JSON eliminates the need to
read entire binary files into RAM at one time, resulting in much more
efficient RAM usage. This fixes
#84, which was the initial impetus
for the format change.

NDJSON allows the reading of message records one at a time, eliminating
the need to use JSON streaming (see
#6), resulting in much simpler and
cleaner code.

Disadvantages:
--------------

The ZIP file format add code complexity.

NDJSON is less common then standard JSON.

NDJSON is less easily humanly-readable than the pretty-printed JSON
previously used (since NDJSON records cannot contain newlines), although
this can be easily mitigated by simply running 'jq < messages.ndjson' to
pretty-print the NDJSON.

Additional Changes:
-------------------

An additional change in this commit is the prefixing of a double
underscore to all (ND)JSON attributes added by the app (e.g.,
'__display_name', '__parts'), in order to clearly indicate that these
have been added by the app and are not the names of columns in the
Android message database tables.

Bugs:
-----

The current implementation of the new format works, although import
performance is unacceptably poor for large message collections. This is
apparently a consequence of the use of the InputStream paradigm
(required by Android's Storage Access framework) to access the ZIP file,
which allows only sequential access, not random access, and so accessing
each binary data file requires a sequential read from the beginning of
the ZIP file. This should be fixed in a subsequent commit.

Closes: #6, #84

Author: tmo1

app/src/main/java/com/github/tmo1/sms_ie/ExportWorker.kt

@@ -1,6 +1,8 @@
 /*
- * SMS Import / Export: a simple Android app for importing and exporting SMS messages from and to JSON files.
- * Copyright (c) 2021-2022 Thomas More
+ * SMS Import / Export: a simple Android app for importing and exporting SMS and MMS messages,
+ * call logs, and contacts, from and to JSON / NDJSON files.
+ *
+ * Copyright (c) 2021-2023 Thomas More
  *
  * This file is part of SMS Import / Export.
  *
@@ -15,7 +17,8 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with SMS Import / Export.  If not, see <https://www.gnu.org/licenses/>.
+ * along with SMS Import / Export.  If not, see <https://www.gnu.org/licenses/>
+ *
  */
 
 package com.github.tmo1.sms_ie
@@ -53,7 +56,7 @@ class ExportWorker(appContext: Context, workerParams: WorkerParameters) :
         CoroutineScope(Dispatchers.IO).launch {
             if (prefs.getBoolean("export_messages", true)) {
                 val file =
-                    documentTree?.createFile("application/json", "messages$dateInString.json")
+                    documentTree?.createFile("application/zip", "messages$dateInString.zip")
                 val fileUri = file?.uri
                 if (fileUri != null) {
                     Log.v(LOG_TAG, "Beginning messages export ...")
@@ -174,10 +177,11 @@ fun deleteOldExports(
         val newFilename = newExport?.name.toString()
         val files = documentTree.listFiles()
         var total = 0
+        val extension = if (prefix == "messages") "zip" else "json"
         files.forEach {
             val name = it.name
             if (name != null && name != newFilename && name.startsWith(prefix) && name.endsWith(
-                    ".json"
+                    ".$extension"
                 )
             ) {
                 it.delete()
@@ -186,7 +190,7 @@ fun deleteOldExports(
         }
         if (prefs.getBoolean("remove_datestamps_from_filenames", false)
         ) {
-            newExport?.renameTo("$prefix.json")
+            newExport?.renameTo("$prefix.$extension")
         }
         Log.v(LOG_TAG, "$total exports deleted")
     }

app/src/main/java/com/github/tmo1/sms_ie/ImportExport.kt

@@ -43,12 +43,9 @@ import kotlinx.coroutines.withContext
 
 fun checkReadSMSContactsPermissions(appContext: Context): Boolean {
     if (ContextCompat.checkSelfPermission(
-            appContext,
-            Manifest.permission.READ_SMS
-        ) == PackageManager.PERMISSION_GRANTED
-        && ContextCompat.checkSelfPermission(
-            appContext,
-            Manifest.permission.READ_CONTACTS
+            appContext, Manifest.permission.READ_SMS
+        ) == PackageManager.PERMISSION_GRANTED && ContextCompat.checkSelfPermission(
+            appContext, Manifest.permission.READ_CONTACTS
         ) == PackageManager.PERMISSION_GRANTED
     ) return true
     /*else {
@@ -63,12 +60,9 @@ fun checkReadSMSContactsPermissions(appContext: Context): Boolean {
 
 fun checkReadCallLogsContactsPermissions(appContext: Context): Boolean {
     if (ContextCompat.checkSelfPermission(
-            appContext,
-            Manifest.permission.READ_CALL_LOG
-        ) == PackageManager.PERMISSION_GRANTED
-        && ContextCompat.checkSelfPermission(
-            appContext,
-            Manifest.permission.READ_CONTACTS
+            appContext, Manifest.permission.READ_CALL_LOG
+        ) == PackageManager.PERMISSION_GRANTED && ContextCompat.checkSelfPermission(
+            appContext, Manifest.permission.READ_CONTACTS
         ) == PackageManager.PERMISSION_GRANTED
     ) return true
     /*else {
@@ -83,66 +77,51 @@ fun checkReadCallLogsContactsPermissions(appContext: Context): Boolean {
 
 fun checkReadWriteCallLogPermissions(appContext: Context): Boolean {
     return ContextCompat.checkSelfPermission(
-        appContext,
-        Manifest.permission.WRITE_CALL_LOG
-    ) == PackageManager.PERMISSION_GRANTED
-            && ContextCompat.checkSelfPermission(
-        appContext,
-        Manifest.permission.READ_CALL_LOG
+        appContext, Manifest.permission.WRITE_CALL_LOG
+    ) == PackageManager.PERMISSION_GRANTED && ContextCompat.checkSelfPermission(
+        appContext, Manifest.permission.READ_CALL_LOG
     ) == PackageManager.PERMISSION_GRANTED
 }
 
 fun checkReadContactsPermission(appContext: Context): Boolean {
     return ContextCompat.checkSelfPermission(
-        appContext,
-        Manifest.permission.READ_CONTACTS
+        appContext, Manifest.permission.READ_CONTACTS
     ) == PackageManager.PERMISSION_GRANTED
 }
 
 fun checkWriteContactsPermission(appContext: Context): Boolean {
     return ContextCompat.checkSelfPermission(
-        appContext,
-        Manifest.permission.WRITE_CONTACTS
+        appContext, Manifest.permission.WRITE_CONTACTS
     ) == PackageManager.PERMISSION_GRANTED
 }
 
 fun lookupDisplayName(
-    appContext: Context,
-    displayNames: MutableMap<String, String?>,
-    address: String
+    appContext: Context, displayNames: MutableMap<String, String?>, address: String
 ): String? {
 //        look up display name by phone number
     if (address == "") return null
     if (displayNames[address] != null) return displayNames[address]
     val displayName: String?
     val uri = Uri.withAppendedPath(
-        ContactsContract.PhoneLookup.CONTENT_FILTER_URI,
-        Uri.encode(address)
+        ContactsContract.PhoneLookup.CONTENT_FILTER_URI, Uri.encode(address)
     )
     val nameCursor = appContext.contentResolver.query(
-        uri,
-        arrayOf(ContactsContract.PhoneLookup.DISPLAY_NAME),
-        null,
-        null,
-        null
+        uri, arrayOf(ContactsContract.PhoneLookup.DISPLAY_NAME), null, null, null
     )
     nameCursor.use {
-        displayName = if (it != null && it.moveToFirst())
-            it.getString(
-                it.getColumnIndexOrThrow(
-                    ContactsContract.PhoneLookup.DISPLAY_NAME
-                )
+        displayName = if (it != null && it.moveToFirst()) it.getString(
+            it.getColumnIndexOrThrow(
+                ContactsContract.PhoneLookup.DISPLAY_NAME
             )
+        )
         else null
     }
     displayNames[address] = displayName
     return displayName
 }
 
 suspend fun wipeSmsAndMmsMessages(
-    appContext: Context,
-    statusReportText: TextView,
-    progressBar: ProgressBar
+    appContext: Context, statusReportText: TextView, progressBar: ProgressBar
 ) {
     val prefs = PreferenceManager.getDefaultSharedPreferences(appContext)
     withContext(Dispatchers.IO) {
@@ -194,14 +173,18 @@ suspend fun incrementProgress(progressBar: ProgressBar?) {
 }
 
 // From: https://stackoverflow.com/a/18143773
-suspend fun displayError(appContext: Context, e: Exception, title: String, message: String) {
-    e.printStackTrace()
+suspend fun displayError(appContext: Context, e: Exception?, title: String, message: String) {
+    val messageExpanded = if (e != null) {
+        e.printStackTrace()
+        "$message:\n\n\"$e\"\n\nSee logcat for more information."
+    } else {
+        message
+    }
     val errorBox = AlertDialog.Builder(appContext)
-    errorBox.setTitle(title)
-        .setMessage("$message:\n\n\"$e\"\n\nSee logcat for more information.")
-        .setCancelable(false)
-        .setNeutralButton("Okay", null)
+    errorBox.setTitle(title).setMessage(messageExpanded)
+    //errorBox.setTitle(title).setMessage("$message:\n\n\"$e\"\n\nSee logcat for more information.")
+        .setCancelable(false).setNeutralButton("Okay", null)
     withContext(Dispatchers.Main) {
         errorBox.show()
     }
-}
+}