Add FXIOS-10667 Content blocklist support via Remote Settings (#23342)

* [FXIOS-10667] Remove the old Disconnect file references from Xcode, as well as old content blocker script, and update the bootstrap to no longer call it

* [FXIOS-10667] Add xcodeproj references to the new Remote Settings default disconnect (blocklist) files

* [FXIOS-10667] Initial work to load blocklist JSON through remote settings APIs rather than directly as bundle resource

* [FXIOX-10667] Comment cleanup

* [FXIOS-10667] Have more generalized RemoteSetting load func call into new JSON utility func.

* [FXIOS-10667] Use RemoteSettings API for hash check for content lists

* [FXIOS-10667] Add and update tests for RemoteDataTypeRecord

* [FXIOS-10667] Fix missing node npm in bootstrap

bootstrap.sh

@@ -47,5 +47,6 @@ cp -r .githooks/* .git/hooks/
 # Make the hooks are executable
 chmod +x .git/hooks/*
 
-# Run and update content blocker
-./content_blocker_update.sh
+# Install Node.js dependencies and build user scripts
+npm install
+npm run build

firefox-ios/Client/Application/RemoteSettings/RemoteDataType.swift

@@ -20,30 +20,62 @@ enum RemoteDataTypeError: Error, LocalizedError {
 
 enum RemoteDataType: String, Codable {
     case passwordRules
+    case contentBlockingLists
 
     var type: any RemoteDataTypeRecord.Type {
         switch self {
         case .passwordRules:
             return PasswordRuleRecord.self
+        case .contentBlockingLists:
+            return ContentBlockingListRecord.self
         }
     }
 
-    var fileName: String {
+    var fileNames: [String] {
         switch self {
         case .passwordRules:
-            return "RemotePasswordRules"
+            return ["RemotePasswordRules"]
+        case .contentBlockingLists:
+            return BlocklistFileName.allCases.map { $0.filename }
         }
     }
 
     var name: String {
         switch self {
         case .passwordRules:
             return "Password Rules"
+        case .contentBlockingLists:
+            return "Content Blocking Lists"
         }
     }
 
+    /// Loads the local settings for the given data type record, returning the
+    /// decoded objects.
+    /// - Returns: settings decoded to their RemoteDataTypeRecord.
     func loadLocalSettingsFromJSON<T: RemoteDataTypeRecord>() async throws -> [T] {
-        let fileName = self.fileName
+        guard let fileName = self.fileNames.first else {
+            assertionFailure("No filename available for setting type.")
+            throw RemoteDataTypeError.fileNotFound(fileName: "")
+        }
+
+        let data = try loadLocalSettingsFileAsJSON(fileName: fileName)
+        do {
+            if let decodedArray = try? JSONDecoder().decode([T].self, from: data) {
+                return decodedArray
+            }
+            let singleObject = try JSONDecoder().decode(T.self, from: data)
+            return [singleObject]
+        } catch {
+            throw RemoteDataTypeError.decodingError(fileName: fileName, error: error)
+        }
+    }
+
+    /// Loads the local settings JSON for the given setting file.
+    /// - Returns: the raw JSON file data.
+    func loadLocalSettingsFileAsJSON(fileName: String) throws -> Data {
+        guard fileNames.contains(fileName) else {
+            throw RemoteDataTypeError.fileNotFound(fileName: fileName)
+        }
 
         guard let path = Bundle.main.path(forResource: fileName, ofType: "json") else {
             throw RemoteDataTypeError.fileNotFound(fileName: fileName)
@@ -53,12 +85,7 @@ enum RemoteDataType: String, Codable {
 
         do {
             let data = try Data(contentsOf: url)
-
-            if let decodedArray = try? JSONDecoder().decode([T].self, from: data) {
-                return decodedArray
-            }
-            let singleObject = try JSONDecoder().decode(T.self, from: data)
-            return [singleObject]
+            return data
         } catch {
             throw RemoteDataTypeError.decodingError(fileName: fileName, error: error)
         }

firefox-ios/Client/Application/RemoteSettings/RemoteRecords/ContentBlockingListRecord.swift

@@ -0,0 +1,11 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/
+
+import Foundation
+
+/// Object model to represent content blocking rules.
+/// This is not used (yet) since we load the JSON directly
+/// in order to modify it before injecting to WKWebView.
+struct ContentBlockingListRecord: RemoteDataTypeRecord {
+}

firefox-ios/Client/ContentBlocker/ContentBlocker.swift

@@ -253,8 +253,14 @@ extension ContentBlocker {
                 let suffixLength = jsonSuffix.count
                 // Trim off .json suffix if needed, we only want the raw file name
                 let fileTrimmed = file.hasSuffix(jsonSuffix) ? String(file.dropLast(suffixLength)) : file
-                if let path = Bundle.main.path(forResource: fileTrimmed, ofType: "json") {
-                    source = try String(contentsOfFile: path, encoding: .utf8)
+
+                if fileTrimmed.hasPrefix(BlocklistFileName.customBlocklistJSONFilePrefix) {
+                    if let path = Bundle.main.path(forResource: fileTrimmed, ofType: "json") {
+                        source = try String(contentsOfFile: path, encoding: .utf8)
+                    }
+                } else {
+                    let json = try RemoteDataType.contentBlockingLists.loadLocalSettingsFileAsJSON(fileName: fileTrimmed)
+                    source = String(data: json, encoding: .utf8) ?? ""
                 }
             } catch let error {
                 logger.log("Error loading content-blocking JSON: \(error)", level: .warning, category: .adblock)
@@ -288,11 +294,7 @@ extension ContentBlocker {
         }
     }
 
-    private func calculateHash(forFileAtPath path: String) -> String? {
-        guard let fileData = try? Data(contentsOf: URL(fileURLWithPath: path)) else {
-            return nil
-        }
-
+    private func calculateHash(for fileData: Data) -> String? {
         let hash = SHA256.hash(data: fileData)
         return hash.compactMap { String(format: "%02x", $0) }.joined()
     }
@@ -302,9 +304,10 @@ extension ContentBlocker {
         let defaults = UserDefaults.standard
         var hasChanged = false
 
+        let lists = RemoteDataType.contentBlockingLists
         for list in blocklists {
-            guard let path = Bundle.main.path(forResource: list, ofType: "json"),
-                  let newHash = calculateHash(forFileAtPath: path) else { continue }
+            guard let data = try? lists.loadLocalSettingsFileAsJSON(fileName: list) else { continue }
+            guard let newHash = calculateHash(for: data) else { continue }
 
             let oldHash = defaults.string(forKey: list)
             if oldHash != newHash {

firefox-ios/Client/ContentBlocker/TrackingProtectionPageStats.swift

@@ -171,13 +171,12 @@ class TPStatsBlocklists {
             ] {
             let list: [[String: AnyObject]]
             do {
-                guard let path = Bundle.main.path(forResource: blockListFile.filename, ofType: "json") else {
-                    logger.log("Blocklists: bad file path.", level: .warning, category: .webview)
-                    assertionFailure("Blocklists: bad file path.")
-                    return
+                let settingsLists = RemoteDataType.contentBlockingLists
+                guard let json = try? settingsLists.loadLocalSettingsFileAsJSON(fileName: blockListFile.filename) else {
+                    logger.log("Blocklists: could not load blocklist JSON file.", level: .warning, category: .webview)
+                    assertionFailure("Blocklists: could not load file.")
+                    continue
                 }
-
-                let json = try Data(contentsOf: URL(fileURLWithPath: path))
                 guard let data = try JSONSerialization.jsonObject(
                     with: json,
                     options: []

firefox-ios/firefox-ios-tests/Tests/ClientTests/RemoteSettings/RemoteDataTypeTests.swift

@@ -51,13 +51,50 @@ class RemoteDataTypeTests: XCTestCase {
         }
     }
 
+    // MARK: ContentBlockingListRecord Tests
+
+    func testLoadContentBlockingListRecords() async throws {
+        // Note: currently ContentBlockingListRecord is a placeholder model.
+        do {
+            let _: [ContentBlockingListRecord] = try await loadAndTestRecords(for: .contentBlockingLists)
+        } catch {
+            XCTFail("testLoadContentBlockingListRecords failed: \(error)")
+        }
+    }
+
+    func testRecordsInContentBlockingJSONFileNotEmpty() async throws {
+        // Note: currently ContentBlockingListRecord is a placeholder model.
+        do {
+            let records: [ContentBlockingListRecord] = try await loadAndTestRecords(for: .contentBlockingLists)
+            XCTAssertGreaterThan(records.count, 0, "Expected more than 0 records, but found none")
+        } catch {
+            XCTFail("testRecordsInContentBlockingJSONFileNotEmpty failed: \(error)")
+        }
+    }
+
+    func testLoadContentBlockListJSONFiles() {
+        let lists = RemoteDataType.contentBlockingLists
+        lists.fileNames.forEach {
+            do {
+                let data = try lists.loadLocalSettingsFileAsJSON(fileName: $0)
+                XCTAssertNotNil(data, "Received nil data for content blocking JSON data. File: \($0).")
+            } catch {
+                XCTFail("Error while attempting to decode content blocking list \($0): \(error)")
+            }
+        }
+    }
+
     // MARK: Helper
 
     // Indirectly tests `loadLocalSettingsFromJSON` by calling it within this function.
     // Any failure in loading or decoding will propagate here and fail the test.
     func loadAndTestRecords<T: RemoteDataTypeRecord>(for remoteDataType: RemoteDataType) async throws -> [T] {
-        guard Bundle(for: type(of: self)).path(forResource: remoteDataType.fileName, ofType: "json") != nil else {
-            XCTFail("\(remoteDataType.fileName).json not found in test bundle")
+        guard let fileName = remoteDataType.fileNames.first else {
+            XCTFail("\(String(describing: remoteDataType)) fileNames list is unexpectedly empty.")
+            return []
+        }
+        guard Bundle(for: type(of: self)).path(forResource: fileName, ofType: "json") != nil else {
+            XCTFail("\(fileName).json not found in test bundle")
             return []
         }
 
@@ -66,7 +103,7 @@ class RemoteDataTypeTests: XCTestCase {
             XCTAssertGreaterThan(records.count, 0, "Expected more than 0 records")
             return records
         } catch {
-            XCTFail("Failed to load and decode records from \(remoteDataType.fileName).json: \(error)")
+            XCTFail("Failed to load and decode records from \(fileName).json: \(error)")
             throw error
         }
     }