Addition of tests. Large commit, one message won't do it justice.

.gitignore

@@ -25,9 +25,14 @@
 *.csv
 *-shm
 *-wal
+*-wal2
 
 ## Stuff I shouldn't index
 passwords.txt
+spec/data/itunes_backup
+spec/data/physical_backup
+spec/data/mac_backup
+spec/data/mac_backup_old
 
 ## Documentation cache and generated files:
 /.yardoc/

.travis.yml

@@ -5,16 +5,7 @@ os:
   - osx
 
 rvm:
-  - "2.3.0"
-  - "2.3.1"
-  - "2.4.3"
-  - "2.5.1"
-  - "2.6.5"
-  - "2.7.1"
-
-jobs:
-  exclude:
-    - os: osx
-      rvm: 2.3.0
-    - os: osx
-      rvm: 2.3.1
+  - "3.0"
+  - "3.1"
+  - "3.2"
+  - "3.3"

FolderStructure.md

@@ -31,9 +31,20 @@ apple_cloud_notes_parser
   |     |-NoteStore.sqlite: If run on a modern version, this copy of the target file will include plaintext versions of the Notes
   |     |-notes.sqlite: If run on a legacy version, this copy is just a copy for ease of use
   |
+  |-spec
+  |  |
+  |  |-backup: Test specs related to Backup classes.
+  |  |-base_classes: Test specs related to foundational classes like AppleNote.
+  |  |-data: Folder container test data. See [this file](spec/data/README.md) for the folder structure if you want to add data.
+  |  |-embedded_objects: Test specs related to EmbeddedObjects. This needs the most work.
+  |  |-integration: Test specs related to the full program running.
+  |  |-output: Test specs related to how output should appear. WARNING, this may change as things are stablized.
+  |  |-spec.rb: The base RSpec file.
+  |  |-utilities: Test specs related to utility classes like AppleDecrypter.
+  |  
   |-.gitignore
   |-.travis.yml
-  |-Dockerfile
+  |-Dockerfile: The Dockerfile for Github's Docker registry
   |-Gemfile
   |-LICENSE
   |-README.md

README.md

@@ -22,6 +22,7 @@ This program will:
 7. ... displaying tables as actual tables and ripping the embedded images from the backup and putting them into a folder with the other output files for review
 8. ... identifying the CloudKit participants involved in any shared items.
 9. ... producing well-structured JSON for automated backups
+10. ... actually run tests against its output after 5 years of YOLO.
 
 ## Usage
 
@@ -243,6 +244,18 @@ On each OS, you will want to:
 5. Use bundler to install the required gems.
 6. Run the program (see Usage section)!
 
+## Tests
+
+As of August 2024, tests have been added using [RSpec](https://rspec.info/). 
+This test suite is not finished and has been in progress for a while, but in order to maintain consistent output with a few key additions, some tests are needed sooner than all tests.
+Because a lot of the test data is inherently sensitive, coming from large Apple Notes backups that contain PII, the tests have been structured to [accept symlinks](spec/data/README.md) and skip tests that require data which is not shareable. 
+While this means that not everyone can benefit from the full test suite, I felt it better to have some data available for tests for PRs rather than keep all of them private. 
+To the extent that data can be extracted and committed into the repo, outside of full backups, that is the preference. 
+
+By default, running `rake test` will skip any tests that are fairly "expensive", primarily in terms of disk IO. 
+If you want to run just the expensive tests, use `rake test_expensive`.
+If you want to run everything, use `rake test_all`. 
+
 ## FAQ
 
 #### Where can I find (pick file X) to edit this?

Rakefile

@@ -1,3 +1,5 @@
+require 'rspec/core'
+
 # By default, execute "run"
 task default: %w[run]
 
@@ -16,3 +18,15 @@ end
 task :clean do 
   FileUtils.rm_rf('output')
 end
+
+task :test do
+  RSpec::Core::Runner.run(["spec/spec.rb", "--tag", "~expensive"])
+end
+
+task :test_expensive do
+  RSpec::Core::Runner.run(["spec/spec.rb", "--tag", "expensive"])
+end
+
+task :test_all do
+  RSpec::Core::Runner.run(["spec/spec.rb", "--tag", "~missing_data"])
+end

lib/ProtoPatches.rb

@@ -68,7 +68,17 @@ def same_style?(other_attribute_run)
 
     no_attachment_info = !attachment_info # We don't want to get so greedy with attachments
 
-    return (same_paragraph and same_font and same_font_weight and same_underlined and same_strikethrough and same_superscript and same_link and same_color and same_attachment_info and no_attachment_info and same_block_quote)
+    return (same_paragraph and 
+            same_font and 
+            same_font_weight and 
+            same_underlined and 
+            same_strikethrough and 
+            same_superscript and 
+            same_link and 
+            same_color and 
+            same_attachment_info and 
+            no_attachment_info and 
+            same_block_quote)
   end
 
   ##
@@ -170,7 +180,9 @@ def is_any_list?
   # it has to recursively check the previous AttributeRuns.
   def total_indent
 
-    to_return = 0
+    return @indent if @indent
+
+    @indent = 0
 
     # Determine what this AttributeRun's indent amount is on its own
     my_indent = 0
@@ -180,13 +192,13 @@ def total_indent
 
     # If there is no previous AttributeRun, the answer is just this AttributeRun's indent amount
     if !previous_run
-      to_return = my_indent
+      @indent = my_indent
     # If there is something previous, add our indent to its total indent
     else
-      to_return = my_indent + previous_run.total_indent
+      @indent = my_indent + previous_run.total_indent
     end
 
-    return to_return
+    return @indent
   end
 
   def open_html_tag(tag_name, attributes = {})

spec/backup/apple_backup.rb

@@ -0,0 +1,25 @@
+require_relative '../../lib/AppleBackup.rb'
+
+describe AppleBackup, :expensive => true do
+  before(:context) do
+    TEST_OUTPUT_DIR.mkpath
+  end
+  after(:context) do 
+    TEST_OUTPUT_DIR.rmtree
+  end
+
+  let(:backup) { AppleBackup.new(TEST_DATA_DIR, 0, TEST_OUTPUT_DIR) }
+
+  context "validations" do
+    it "raises an error rather than failing validation" do
+      expect{backup.valid?}.to raise_error("AppleBackup cannot stand on its own")
+    end
+
+  end
+
+  context "files" do 
+    it "raises an error since it has no real file path" do
+      expect{backup.get_real_file_path("test.tmp")}.to raise_error("Cannot return file_path for AppleBackup")
+    end
+  end
+end

spec/backup/apple_backup_file.rb

@@ -0,0 +1,67 @@
+require_relative '../../lib/AppleBackupFile.rb'
+
+describe AppleBackupFile, :expensive => true do
+  before(:context) do
+    TEST_OUTPUT_DIR.mkpath
+  end
+  after(:context) do 
+    TEST_OUTPUT_DIR.rmtree
+  end
+
+  let(:valid_backup) { AppleBackupFile.new(TEST_FORMATTING_FILE, TEST_OUTPUT_DIR) }
+
+  context "validations" do
+    it "validates a NoteStore.sqlite file", :missing_data => !TEST_FORMATTING_FILE_EXIST do
+      expect(valid_backup.valid?).to be true
+    end
+
+    it "fails to validate a non-NoteStore sqlite file", :missing_data => !TEST_FALSE_SQLITE_FILE_EXIST do
+      backup = AppleBackupFile.new(TEST_FALSE_SQLITE_FILE, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a non-sqlite file", :missing_data => !TEST_README_FILE_EXIST do
+      backup = AppleBackupFile.new(TEST_README_FILE, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate an itunes backup folder", :missing_data => !TEST_ITUNES_DIR_EXIST do
+      backup = AppleBackupFile.new(TEST_ITUNES_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a physical backup folder", :missing_data => !TEST_PHYSICAL_DIR_EXIST do
+      backup = AppleBackupFile.new(TEST_PHYSICAL_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a mac backup folder", :missing_data => !TEST_MAC_DIR_EXIST do
+      backup = AppleBackupFile.new(TEST_MAC_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+  end
+
+  context "versions" do
+
+    it "correctly identifies all major versions" do
+      # To do: acquire iOS 11 sample for here
+      TEST_FILE_VERSIONS.each_pair do |version, version_file|
+        backup = AppleBackupFile.new(version_file, TEST_OUTPUT_DIR)
+        expect(backup.note_stores[0].version.version_number).to be version
+      end
+    end
+  end
+
+  context "files", :missing_data => !TEST_FORMATTING_FILE_EXIST do 
+    it "does not try to assert where a file is" do
+      backup = AppleBackupFile.new(TEST_FORMATTING_FILE, TEST_OUTPUT_DIR)
+      expect(valid_backup.get_real_file_path("NoteStore.sqlite")).to be nil
+    end
+  end
+
+  context "note stores", :missing_data => !TEST_FORMATTING_FILE_EXIST do
+    it "knows how to find just a modern note store" do 
+      expect(valid_backup.note_stores.length).to be 1
+    end
+  end
+end

spec/backup/apple_backup_hashed.rb

@@ -0,0 +1,45 @@
+require_relative '../../lib/AppleBackupHashed.rb'
+
+describe AppleBackupHashed, :expensive => true do
+  before(:context) do
+    TEST_OUTPUT_DIR.mkpath
+  end
+  after(:context) do 
+    TEST_OUTPUT_DIR.rmtree
+  end
+
+  let(:valid_backup) { AppleBackupHashed.new(TEST_ITUNES_DIR, TEST_OUTPUT_DIR) }
+
+  context "validations" do
+    it "validates an itunes backup folder", :missing_data => !TEST_ITUNES_DIR_EXIST do
+      expect(valid_backup.valid?).to be true
+    end
+
+    it "fails to validate a physical backup folder", :missing_data => !TEST_PHYSICAL_DIR_EXIST do
+      backup = AppleBackupHashed.new(TEST_PHYSICAL_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a mac backup folder", :missing_data => !TEST_MAC_DIR_EXIST do
+      backup = AppleBackupHashed.new(TEST_MAC_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a valid NoteStore.sqlite file", :missing_data => !TEST_FORMATTING_FILE_EXIST do
+      backup = AppleBackupHashed.new(TEST_FORMATTING_FILE, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+  end
+
+  context "files", :missing_data => !TEST_ITUNES_DIR_EXIST do 
+    it "knows how to find an appropriate file" do
+      expect(valid_backup.get_real_file_path("NoteStore.sqlite").to_s).to match(/spec\/data\/itunes_backup\/4f\/4f98687d8ab0d6d1a371110e6b7300f6e465bef2/)
+    end
+  end
+
+  context "note stores", :missing_data => !TEST_ITUNES_DIR_EXIST do
+    it "knows how to find both legacy and modern note stores" do 
+      expect(valid_backup.note_stores.length).to be 2
+    end
+  end
+end

spec/backup/apple_backup_mac.rb

@@ -0,0 +1,45 @@
+require_relative '../../lib/AppleBackupMac.rb'
+
+describe AppleBackupMac, :expensive => true do
+  before(:context) do
+    TEST_OUTPUT_DIR.mkpath
+  end
+  after(:context) do 
+    TEST_OUTPUT_DIR.rmtree
+  end
+
+  let(:valid_backup) { AppleBackupMac.new(TEST_MAC_DIR, TEST_OUTPUT_DIR) }
+
+  context "validations" do
+    it "validates a mac backup folder", :missing_data => !TEST_MAC_DIR_EXIST do
+      expect(valid_backup.valid?).to be true
+    end
+
+    it "fails to validate an itunes backup folder", :missing_data => !TEST_ITUNES_DIR_EXIST do
+      backup = AppleBackupMac.new(TEST_ITUNES_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a physical backup folder", :missing_data => !TEST_PHYSICAL_DIR_EXIST do
+      backup = AppleBackupMac.new(TEST_PHYSICAL_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a valid NoteStore.sqlite file", :missing_data => !TEST_FORMATTING_FILE_EXIST do
+      backup = AppleBackupMac.new(TEST_FORMATTING_FILE, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+  end
+
+  context "files", :missing_data => !TEST_MAC_DIR_EXIST do 
+    it "knows how to find an appropriate file" do
+      expect(valid_backup.get_real_file_path("NoteStore.sqlite").to_s).to match(/spec\/data\/mac_backup\/NoteStore.sqlite/)
+    end
+  end
+
+  context "note stores", :missing_data => !TEST_MAC_DIR_EXIST do
+    it "knows how to find just a modern note store" do 
+      expect(valid_backup.note_stores.length).to be 1
+    end
+  end
+end

spec/backup/apple_backup_physical.rb

@@ -0,0 +1,45 @@
+require_relative '../../lib/AppleBackupPhysical.rb'
+
+describe AppleBackupPhysical, :expensive => true do
+  before(:context) do
+    TEST_OUTPUT_DIR.mkpath
+  end
+  after(:context) do 
+    TEST_OUTPUT_DIR.rmtree
+  end
+
+  let(:valid_backup) { AppleBackupPhysical.new(TEST_PHYSICAL_DIR, TEST_OUTPUT_DIR) }
+
+  context "validations" do
+    it "validates a physical backup folder", :missing_data => !TEST_PHYSICAL_DIR_EXIST do
+      expect(valid_backup.valid?).to be true
+    end
+
+    it "fails to validate an itunes backup folder", :missing_data => !TEST_ITUNES_DIR_EXIST do
+      backup = AppleBackupPhysical.new(TEST_ITUNES_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a mac backup folder", :missing_data => !TEST_MAC_DIR_EXIST do
+      backup = AppleBackupPhysical.new(TEST_MAC_DIR, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+
+    it "fails to validate a valid NoteStore.sqlite file", :missing_data => !TEST_FORMATTING_FILE_EXIST do
+      backup = AppleBackupPhysical.new(TEST_FORMATTING_FILE, TEST_OUTPUT_DIR)
+      expect(backup.valid?).to be false
+    end
+  end
+
+  context "files", :missing_data => !TEST_PHYSICAL_DIR_EXIST do 
+    it "knows how to find an appropriate file" do
+      expect(valid_backup.get_real_file_path("NoteStore.sqlite").to_s).to match(/private\/var\/mobile\/Containers\/Shared\/AppGroup\/[A-F0-9\-]{36}\/NoteStore.sqlite/)
+    end
+  end
+
+  context "note stores", :missing_data => !TEST_PHYSICAL_DIR_EXIST do
+    it "knows how to find both legacy and modern note stores" do 
+      expect(valid_backup.note_stores.length).to be 2
+    end
+  end
+end

spec/backup/backup.rb

@@ -0,0 +1,5 @@
+require_relative 'apple_backup.rb'
+require_relative 'apple_backup_file.rb'
+require_relative 'apple_backup_hashed.rb'
+require_relative 'apple_backup_mac.rb'
+require_relative 'apple_backup_physical.rb'