feat: use Tiktokenizer to count tokens

2025-07-23 21:13:50 +00:00 · 2023-04-24 15:18:05 +08:00
parent 29c95d24ae
commit adbaa8b37b
8 changed files with 149 additions and 32 deletions
--- a/next.config.js
+++ b/next.config.js
@@ -4,7 +4,12 @@ const nextConfig = {
  output: 'standalone',
  reactStrictMode: true,
  compress: true,
+
  webpack(config) {
+    config.experiments = {
+      asyncWebAssembly: true,
+      layers: true
+    };
    config.module.rules = config.module.rules.concat([
      {
        test: /\.svg$/i,