Bump swift-transformers to 0.1.12

argmaxinc · Nov 4, 2024 · 8f553ba · 8f553ba
1 parent dd2eb73
commit 8f553ba
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 10 deletions.
diff --git a/.swiftpm/configuration/Package.resolved b/.swiftpm/configuration/Package.resolved
@@ -1,21 +1,30 @@
 {
   "pins" : [
+    {
+      "identity" : "jinja",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/maiqingqiang/Jinja",
+      "state" : {
+        "revision" : "6dbe4c449469fb586d0f7339f900f0dd4d78b167",
+        "version" : "1.0.6"
+      }
+    },
     {
       "identity" : "swift-argument-parser",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-argument-parser.git",
       "state" : {
-        "revision" : "c8ed701b513cf5177118a175d85fbbbcd707ab41",
-        "version" : "1.3.0"
+        "revision" : "0fbc8848e389af3bb55c182bc19ca9d5dc2f255b",
+        "version" : "1.4.0"
       }
     },
     {
       "identity" : "swift-transformers",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/huggingface/swift-transformers.git",
       "state" : {
-        "revision" : "74b94211bdc741694ed7e700a1104c72e5ba68fe",
-        "version" : "0.1.7"
+        "revision" : "0f2306713d48a75b862026ebb291926793773f52",
+        "version" : "0.1.12"
       }
     }
   ],

diff --git a/Package.resolved b/Package.resolved
@@ -1,21 +1,30 @@
 {
   "pins" : [
+    {
+      "identity" : "jinja",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/maiqingqiang/Jinja",
+      "state" : {
+        "revision" : "6dbe4c449469fb586d0f7339f900f0dd4d78b167",
+        "version" : "1.0.6"
+      }
+    },
     {
       "identity" : "swift-argument-parser",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-argument-parser.git",
       "state" : {
-        "revision" : "c8ed701b513cf5177118a175d85fbbbcd707ab41",
-        "version" : "1.3.0"
+        "revision" : "0fbc8848e389af3bb55c182bc19ca9d5dc2f255b",
+        "version" : "1.4.0"
       }
     },
     {
       "identity" : "swift-transformers",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/huggingface/swift-transformers.git",
       "state" : {
-        "revision" : "fc6543263e4caed9bf6107466d625cfae9357f08",
-        "version" : "0.1.8"
+        "revision" : "0f2306713d48a75b862026ebb291926793773f52",
+        "version" : "0.1.12"
       }
     }
   ],

diff --git a/Package.swift b/Package.swift
@@ -20,8 +20,8 @@ let package = Package(
         ),
     ],
     dependencies: [
-        .package(url: "https://github.com/huggingface/swift-transformers.git", exact: "0.1.8"),
-        .package(url: "https://github.com/apple/swift-argument-parser.git", exact: "1.3.0"),
+        .package(url: "https://github.com/huggingface/swift-transformers.git", exact: "0.1.12"),
+        .package(url: "https://github.com/apple/swift-argument-parser.git", exact: "1.4.0"),
     ],
     targets: [
         .target(

diff --git a/Sources/WhisperKit/Core/Models.swift b/Sources/WhisperKit/Core/Models.swift
@@ -1272,10 +1272,18 @@ extension WhisperTokenizerWrapper: Tokenizer {
     func tokenize(text: String) -> [String] {
         tokenizer.tokenize(text: text)
     }
+
+    func callAsFunction(_ text: String, addSpecialTokens: Bool) -> [Int] {
+        tokenizer.callAsFunction(text, addSpecialTokens: addSpecialTokens)
+    }
 
     func encode(text: String) -> [Int] {
         tokenizer.encode(text: text)
     }
+
+    func encode(text: String, addSpecialTokens: Bool) -> [Int] {
+        tokenizer.encode(text: text, addSpecialTokens: addSpecialTokens)
+    }
 
     func decode(tokens: [Int]) -> String {
         tokenizer.decode(tokens: tokens)
@@ -1312,6 +1320,27 @@ extension WhisperTokenizerWrapper: Tokenizer {
     var unknownTokenId: Int? {
         tokenizer.unknownTokenId
     }
+
+    func applyChatTemplate(messages: [[String: String]]) throws -> [Int] {
+        try tokenizer.applyChatTemplate(messages: messages)
+    }
+
+    func applyChatTemplate(
+        messages: [[String: String]],
+        chatTemplate: String?,
+        addGenerationPrompt: Bool,
+        truncation: Bool,
+        maxLength: Int?
+    ) throws -> [Int] {
+        try tokenizer
+            .applyChatTemplate(
+                messages: messages,
+                chatTemplate: chatTemplate,
+                addGenerationPrompt: addGenerationPrompt,
+                truncation: truncation,
+                maxLength: maxLength
+            )
+    }
 }
 
 extension WhisperTokenizerWrapper {