Merge pull request #168 from ReagentX/develop

Checker Mallow

Cargo.toml

@@ -1,4 +1,6 @@
 [workspace]
+edition = "2021"
+resolver = "2"
 members = [
   "imessage-database",
   "imessage-exporter",

README.md

@@ -18,7 +18,7 @@ Documentation for the library is located [here](imessage-database/README.md).
 
 ### Supported Features
 
-This crate supports every iMessage feature as of MacOS 13.5 (22G74) and iOS 16.6 (20G75):
+This crate supports every iMessage feature as of macOS 13.5.2 (22G91) and iOS 16.6.1 (20G81):
 
 - Multi-part messages
 - Replies/Threads

docs/features.md

@@ -4,7 +4,7 @@ This exporter is fully-featured and well-documented.
 
 ## Targeted Versions
 
-This tool targets the current latest public release for MacOS and iMessage. It may work with older databases, but all features may not be available.
+This tool targets the current latest public release for macOS and iMessage. It may work with older databases, but all features may not be available.
 
 ## Supported Message Features
 

imessage-database/README.md

@@ -38,7 +38,7 @@ fn iter_messages() -> Result<(), TableError> {
     for message in messages {
         let mut msg = Message::extract(message)?;
 
-        /// Parse message body if it was sent from MacOS 13.0 or newer
+        /// Parse message body if it was sent from macOS 13.0 or newer
         msg.gen_text(&db);
 
         /// Emit debug info for each message

imessage-database/src/tables/attachment.rs

@@ -19,7 +19,12 @@ use crate::{
     },
 };
 
+const DIVISOR: f64 = 1024.;
+const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];
+
 /// Represents the MIME type of a message's attachment data
+///
+/// The interior `str` contains the subtype, i.e. `x-m4a` for `audio/x-m4a`
 #[derive(Debug, PartialEq, Eq)]
 pub enum MediaType<'a> {
     Image(&'a str),
@@ -36,9 +41,10 @@ pub enum MediaType<'a> {
 pub struct Attachment {
     pub rowid: i32,
     pub filename: Option<String>,
+    pub uti: Option<String>,
     pub mime_type: Option<String>,
     pub transfer_name: Option<String>,
-    pub total_bytes: i32,
+    pub total_bytes: i64,
     pub hide_attachment: i32,
     pub copied_path: Option<PathBuf>,
 }
@@ -48,6 +54,7 @@ impl Table for Attachment {
         Ok(Attachment {
             rowid: row.get("rowid")?,
             filename: row.get("filename").unwrap_or(None),
+            uti: row.get("uti").unwrap_or(None),
             mime_type: row.get("mime_type").unwrap_or(None),
             transfer_name: row.get("transfer_name").unwrap_or(None),
             total_bytes: row.get("total_bytes").unwrap_or_default(),
@@ -114,7 +121,19 @@ impl Attachment {
                     MediaType::Other(mime)
                 }
             }
-            None => MediaType::Unknown,
+            None => {
+                // Fallback to `uti` if the MIME type cannot be inferred
+                if let Some(uti) = &self.uti {
+                    match uti.as_str() {
+                        // This type is for audio messages, which are sent in `caf` format
+                        // https://developer.apple.com/library/archive/documentation/MusicAudio/Reference/CAFSpec/CAF_overview/CAF_overview.html
+                        "com.apple.coreaudio-format" => MediaType::Audio("x-caf; codecs=opus"),
+                        _ => MediaType::Unknown,
+                    }
+                } else {
+                    MediaType::Unknown
+                }
+            }
         }
     }
 
@@ -148,15 +167,32 @@ impl Attachment {
         "Attachment missing name metadata!"
     }
 
+    /// Get a human readable file size for an attachment
+    pub fn file_size(&self) -> String {
+        Attachment::format_file_size(self.total_bytes)
+    }
+
+    /// Get a human readable file size for an arbitrary amount of bytes
+    fn format_file_size(total_bytes: i64) -> String {
+        let mut index: usize = 0;
+        let mut bytes = total_bytes as f64;
+        while index < UNITS.len() - 1 && bytes > DIVISOR {
+            index += 1;
+            bytes /= DIVISOR;
+        }
+
+        format!("{bytes:.2} {}", UNITS[index])
+    }
+
     /// Given a platform and database source, resolve the path for the current attachment
     ///
-    /// For MacOS, `db_path` is unused. For iOS, `db_path` is the path to the root of the backup directory.
+    /// For macOS, `db_path` is unused. For iOS, `db_path` is the path to the root of the backup directory.
     ///
     /// iOS Parsing logic source is from [here](https://github.com/nprezant/iMessageBackup/blob/940d001fb7be557d5d57504eb26b3489e88de26e/imessage_backup_tools.py#L83-L85).
     pub fn resolved_attachment_path(&self, platform: &Platform, db_path: &Path) -> Option<String> {
         if let Some(path_str) = &self.filename {
             return match platform {
-                Platform::MacOS => Some(Attachment::gen_macos_attachment(path_str)),
+                Platform::macOS => Some(Attachment::gen_macos_attachment(path_str)),
                 Platform::iOS => Attachment::gen_ios_attachment(path_str, db_path),
             };
         }
@@ -182,7 +218,7 @@ impl Attachment {
     ///
     /// let db_path = default_db_path();
     /// let conn = get_connection(&db_path).unwrap();
-    /// Attachment::run_diagnostic(&conn, &db_path, &Platform::MacOS);
+    /// Attachment::run_diagnostic(&conn, &db_path, &Platform::macOS);
     /// ```
     pub fn run_diagnostic(
         db: &Connection,
@@ -206,7 +242,7 @@ impl Attachment {
                 total_attachments += 1;
                 if let Ok(filepath) = path {
                     match platform {
-                        Platform::MacOS => {
+                        Platform::macOS => {
                             !Path::new(&Attachment::gen_macos_attachment(filepath)).exists()
                         }
                         Platform::iOS => {
@@ -227,13 +263,22 @@ impl Attachment {
             })
             .count();
 
+        let mut bytes_query = db
+            .prepare(&format!("SELECT SUM(total_bytes) FROM {ATTACHMENT}"))
+            .map_err(TableError::Messages)?;
+
+        let total_bytes: i64 = bytes_query.query_row([], |r| r.get(0)).unwrap_or(0);
+
         done_processing();
 
-        if missing_files > 0 {
+        if total_attachments > 0 {
             println!("\rAttachment diagnostic data:");
-
+            println!("    Total attachments: {total_attachments}");
+            println!(
+                "    Total attachment data: {}",
+                Attachment::format_file_size(total_bytes)
+            );
             if missing_files > 0 && total_attachments > 0 {
-                println!("    Total attachments: {total_attachments}");
                 println!(
                     "    Missing files: {missing_files:?} ({:.0}%)",
                     (missing_files as f64 / total_attachments as f64) * 100f64
@@ -248,10 +293,10 @@ impl Attachment {
         Ok(())
     }
 
-    /// Generate a MacOS path for an attachment
+    /// Generate a macOS path for an attachment
     fn gen_macos_attachment(path: &str) -> String {
         if path.starts_with('~') {
-            return path.replace('~', &home());
+            return path.replacen('~', &home(), 1);
         }
         path.to_string()
     }
@@ -282,6 +327,7 @@ mod tests {
         Attachment {
             rowid: 1,
             filename: Some("a/b/c.png".to_string()),
+            uti: Some("public.png".to_string()),
             mime_type: Some("image".to_string()),
             transfer_name: Some("c.png".to_string()),
             total_bytes: 100,
@@ -370,7 +416,7 @@ mod tests {
         let attachment = sample_attachment();
 
         assert_eq!(
-            attachment.resolved_attachment_path(&Platform::MacOS, &db_path),
+            attachment.resolved_attachment_path(&Platform::macOS, &db_path),
             Some("a/b/c.png".to_string())
         );
     }
@@ -383,13 +429,25 @@ mod tests {
 
         assert!(
             attachment
-                .resolved_attachment_path(&Platform::MacOS, &db_path)
+                .resolved_attachment_path(&Platform::macOS, &db_path)
                 .unwrap()
                 .len()
                 > attachment.filename.unwrap().len()
         );
     }
 
+    #[test]
+    fn can_get_resolved_path_macos_raw_tilde() {
+        let db_path = PathBuf::from("fake_root");
+        let mut attachment = sample_attachment();
+        attachment.filename = Some("~/a/b/c~d.png".to_string());
+
+        assert!(attachment
+            .resolved_attachment_path(&Platform::macOS, &db_path)
+            .unwrap()
+            .ends_with("c~d.png"));
+    }
+
     #[test]
     fn can_get_resolved_path_ios() {
         let db_path = PathBuf::from("fake_root");
@@ -408,7 +466,7 @@ mod tests {
         attachment.filename = None;
 
         assert_eq!(
-            attachment.resolved_attachment_path(&Platform::MacOS, &db_path),
+            attachment.resolved_attachment_path(&Platform::macOS, &db_path),
             None
         );
     }
@@ -424,4 +482,43 @@ mod tests {
             None
         );
     }
+
+    #[test]
+    fn can_get_file_size_bytes() {
+        let attachment = sample_attachment();
+
+        assert_eq!(attachment.file_size(), String::from("100.00 B"));
+    }
+
+    #[test]
+    fn can_get_file_size_kb() {
+        let mut attachment = sample_attachment();
+        attachment.total_bytes = 2300;
+
+        assert_eq!(attachment.file_size(), String::from("2.25 KB"));
+    }
+
+    #[test]
+    fn can_get_file_size_mb() {
+        let mut attachment = sample_attachment();
+        attachment.total_bytes = 5612000;
+
+        assert_eq!(attachment.file_size(), String::from("5.35 MB"));
+    }
+
+    #[test]
+    fn can_get_file_size_gb() {
+        let mut attachment: Attachment = sample_attachment();
+        attachment.total_bytes = 9234712394;
+
+        assert_eq!(attachment.file_size(), String::from("8.60 GB"));
+    }
+
+    #[test]
+    fn can_get_file_size_cap() {
+        let mut attachment: Attachment = sample_attachment();
+        attachment.total_bytes = i64::MAX;
+
+        assert_eq!(attachment.file_size(), String::from("8388608.00 TB"));
+    }
 }

imessage-database/src/tables/messages.rs

@@ -77,7 +77,7 @@ pub struct Message {
     pub guid: String,
     pub text: Option<String>,
     pub service: Option<String>,
-    pub handle_id: i32,
+    pub handle_id: Option<i32>,
     pub subject: Option<String>,
     pub date: i64,
     pub date_read: i64,
@@ -107,7 +107,7 @@ impl Table for Message {
             guid: row.get("guid")?,
             text: row.get("text").unwrap_or(None),
             service: row.get("service").unwrap_or(None),
-            handle_id: row.get("handle_id")?,
+            handle_id: row.get("handle_id").unwrap_or(None),
             subject: row.get("subject").unwrap_or(None),
             date: row.get("date")?,
             date_read: row.get("date_read").unwrap_or(0),
@@ -132,9 +132,10 @@ impl Table for Message {
     }
 
     fn get(db: &Connection) -> Result<Statement, TableError> {
+        // If the database has `chat_recoverable_message_join`, we can restore some deleted messages.
         // If database has `thread_originator_guid`, we can parse replies, otherwise default to 0
-        // The first sql statement is the "current" schema, the second one is the "most compatible" schema, i.e. supports older DBs
         Ok(db.prepare(&format!(
+            // macOS Ventura+ and i0S 16+ schema
             "SELECT
                  *,
                  c.chat_id,
@@ -147,14 +148,29 @@ impl Table for Message {
              ORDER BY
                  m.date;
             "
-        ))
+        )).or(db.prepare(&format!(
+            // macOS Big Sur to Monterey, iOS 14 to iOS 15 schema
+            "SELECT
+                 *,
+                 c.chat_id,
+                 (SELECT COUNT(*) FROM {MESSAGE_ATTACHMENT_JOIN} a WHERE m.ROWID = a.message_id) as num_attachments,
+                 NULL as deleted_from,
+                 (SELECT COUNT(*) FROM {MESSAGE} m2 WHERE m2.thread_originator_guid = m.guid) as num_replies
+             FROM
+                 message as m
+                 LEFT JOIN {CHAT_MESSAGE_JOIN} as c ON m.ROWID = c.message_id
+             ORDER BY
+                 m.date;
+            "
+        )))
         .unwrap_or(db.prepare(&format!(
+            // macOS Catalina, iOS 13 and older 
             "SELECT
                  *,
                  c.chat_id,
                  (SELECT COUNT(*) FROM {MESSAGE_ATTACHMENT_JOIN} a WHERE m.ROWID = a.message_id) as num_attachments,
-                 (SELECT NULL) as deleted_from,
-                 (SELECT 0) as num_replies
+                 NULL as deleted_from,
+                 0 as num_replies
              FROM
                  message as m
                  LEFT JOIN {CHAT_MESSAGE_JOIN} as c ON m.ROWID = c.message_id
@@ -316,7 +332,7 @@ impl Cacheable for Message {
 
 impl Message {
     /// Get the body text of a message, parsing it as [`streamtyped`](crate::util::streamtyped) data if necessary.
-    /// TODO: resolve the compiler warnings with this method
+    // TODO: resolve the compiler warnings with this method
     pub fn gen_text<'a>(&'a mut self, db: &'a Connection) -> Result<&'a str, MessageError> {
         if self.text.is_none() {
             let body = self.attributed_body(db).ok_or(MessageError::MissingData)?;
@@ -505,6 +521,14 @@ impl Message {
     }
 
     /// `true` if the message was deleted and is recoverable, else `false`
+    ///
+    /// Messages removed by deleting an entire conversation or by deleting a single message
+    /// from a conversation are moved to a separate collection for up to 30 days. Messages
+    /// present in this collection are restored to the conversations they belong to. Apple
+    /// details this process [here](https://support.apple.com/en-us/HT202549#delete).
+    ///
+    /// Messages that have expired from this restoration process are permanently deleted and
+    /// cannot be recovered.
     pub fn is_deleted(&self) -> bool {
         self.deleted_from.is_some()
     }
@@ -934,7 +958,7 @@ mod tests {
             guid: String::default(),
             text: None,
             service: Some("iMessage".to_string()),
-            handle_id: i32::default(),
+            handle_id: Some(i32::default()),
             subject: None,
             date: i64::default(),
             date_read: i64::default(),

imessage-database/src/tables/mod.rs

@@ -2,7 +2,7 @@
 Representations of iMessage database tables as structs.
 
 Many of these tables do not include all available columns. Even on the same versions
-of MacOS, the schema of the iMessage database can vary.
+of macOS, the schema of the iMessage database can vary.
 */
 
 pub mod attachment;

imessage-database/src/tables/table.rs

@@ -92,7 +92,7 @@ pub const ME: &str = "Me";
 pub const YOU: &str = "You";
 /// Name used for contacts or chats where the name cannot be discovered
 pub const UNKNOWN: &str = "Unknown";
-/// Default location for the Messages database on MacOS
+/// Default location for the Messages database on macOS
 pub const DEFAULT_PATH_MACOS: &str = "Library/Messages/chat.db";
 /// Default location for the Messages database in an unencrypted iOS backup
 pub const DEFAULT_PATH_IOS: &str = "3d/3d0d7e5fb2ce288813306e4d4636395e047a3d28";