feat: tokenizers (yetone#407)

* feat: autobuild tiktoken lib and schenanigans Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: revert readme changes Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(build): windows Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz> * chore(plugin): early load commands and base setup Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(build): make sync Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * feat: rust go vroom vroom Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * feat: scuffed afaf implementation binding go brrrr Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: remove dups Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * fix(tokens): calculate whether we should do prompt_caching (fixes yetone#416) Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: ignore lockfiles Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * Update README.md * Update crates/avante-tokenizers/README.md * chore: remove unused Signed-off-by: Aaron Pham <contact@aarnphm.xyz> * chore: remove auto build Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz> Co-authored-by: yuchanns <me@yuchanns.xyz>
icodingforfood · Aug 31, 2024 · d2095ba · d2095ba
1 parent 81b44e4
commit d2095ba
Show file tree

Hide file tree

Showing 27 changed files with 3,554 additions and 27 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,5 @@
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+* text=auto eol=lf
+**/*.lock linguist-generated=true
diff --git a/.gitignore b/.gitignore
@@ -43,5 +43,4 @@ temp/
 # If you have any personal configuration files, you should ignore them too
 config.personal.lua
 
-# Avante chat history
-.avante_chat_history
+target
diff --git a/Build.ps1 b/Build.ps1
@@ -0,0 +1,29 @@
+param (
+    [string]$Version = "luajit"
+)
+
+$BuildDir = "build"
+$BuildFromSource = $true
+
+function Build-FromSource($feature) {
+    if (-not (Test-Path $BuildDir)) {
+        New-Item -ItemType Directory -Path $BuildDir | Out-Null
+    }
+
+    cargo build --release --features=$feature
+
+    $targetFile = "avante_tokenizers.dll"
+    Copy-Item (Join-Path "target\release\libavante_tokenizers.dll") (Join-Path $BuildDir $targetFile)
+
+    Remove-Item -Recurse -Force "target"
+}
+
+function Main {
+    Set-Location $PSScriptRoot
+    Write-Host "Building for $Version..."
+    Build-FromSource $Version
+    Write-Host "Completed!"
+}
+
+# Run the main function
+Main
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		* text=auto eol=lf
		*/.lock linguist-generated=true