feat: tokenizers (yetone#429)

* feat: tokenizers This reverts commit d5a4db8. * fix(inputs): yetone#422 Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
frm · Aug 31, 2024 · 0557dee · 0557dee
1 parent 534b1e6
commit 0557dee
Show file tree

Hide file tree

Showing 28 changed files with 3,553 additions and 27 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,5 @@
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+* text=auto eol=lf
+**/*.lock linguist-generated=true
diff --git a/.gitignore b/.gitignore
@@ -43,5 +43,4 @@ temp/
 # If you have any personal configuration files, you should ignore them too
 config.personal.lua
 
-# Avante chat history
-.avante_chat_history
+target
diff --git a/Build.ps1 b/Build.ps1
@@ -0,0 +1,29 @@
+param (
+    [string]$Version = "luajit"
+)
+
+$BuildDir = "build"
+$BuildFromSource = $true
+
+function Build-FromSource($feature) {
+    if (-not (Test-Path $BuildDir)) {
+        New-Item -ItemType Directory -Path $BuildDir | Out-Null
+    }
+
+    cargo build --release --features=$feature
+
+    $targetFile = "avante_tokenizers.dll"
+    Copy-Item (Join-Path "target\release\libavante_tokenizers.dll") (Join-Path $BuildDir $targetFile)
+
+    Remove-Item -Recurse -Force "target"
+}
+
+function Main {
+    Set-Location $PSScriptRoot
+    Write-Host "Building for $Version..."
+    Build-FromSource $Version
+    Write-Host "Completed!"
+}
+
+# Run the main function
+Main
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		* text=auto eol=lf
		*/.lock linguist-generated=true