From 40168c56ea7b67e0f94a5567be3f3fa21882d302 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Thu, 17 Aug 2023 23:19:19 +0800 Subject: [PATCH] perf: logger (#186) * feat: finish response * perf: logger * docs * perf: log * docs --- client/.env.template | 9 +- client/package.json | 2 + client/pnpm-lock.yaml | 285 +++++++++++++++++- client/src/constants/flow/ModuleTemplate.ts | 16 +- .../pages/api/openapi/v1/chat/completions.ts | 5 +- .../AdEdit/components/Nodes/NodeAnswer.tsx | 2 + client/src/service/events/generateQA.ts | 2 +- client/src/service/events/generateVector.ts | 2 +- client/src/service/events/pushBill.ts | 21 +- .../src/service/moduleDispatch/chat/oneapi.ts | 32 +- .../service/moduleDispatch/tools/answer.ts | 4 +- client/src/service/mongo.ts | 37 ++- client/src/service/pg.ts | 8 +- client/src/service/response.ts | 40 ++- client/src/service/utils/tools.ts | 11 + client/src/types/index.d.ts | 4 + docSite/docs/develop/deploy/docker.md | 24 +- .../{deploy/v4init.md => update/40init.md} | 2 +- .../{deploy/initv4.1.md => update/41init.md} | 9 + docSite/docusaurus.config.js | 3 +- .../current.json | 3 + .../current/develop/deploy/docker.md | 24 +- .../{deploy/v4init.md => update/40init.md} | 2 +- .../{deploy/initv4.1.md => update/41init.md} | 9 + docSite/sidebars.js | 15 +- files/deploy/fastgpt/docker-compose.yml | 22 +- 26 files changed, 500 insertions(+), 93 deletions(-) rename docSite/docs/develop/{deploy/v4init.md => update/40init.md} (98%) rename docSite/docs/develop/{deploy/initv4.1.md => update/41init.md} (55%) rename docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/{deploy/v4init.md => update/40init.md} (98%) rename docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/{deploy/initv4.1.md => update/41init.md} (55%) diff --git a/client/.env.template b/client/.env.template index e2683169b..65f31c643 100644 --- a/client/.env.template +++ b/client/.env.template @@ -17,10 +17,5 @@ OPENAI_BASE_URL=https://api.openai.com/v1 # 此处逻辑：优先走 ONEAPI_URL，如果填写了 ONEAPI_URL，key 也需要是 ONEAPI 的 key CHAT_API_KEY=sk-xxxx # db -MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin -MONGODB_NAME=fastgpt -PG_HOST=0.0.0.0 -PG_PORT=8100 -PG_USER=root -PG_PASSWORD=psw -PG_DB_NAME=dbname \ No newline at end of file +MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt +PG_URL=postgresql://username:password@host:port/postgres \ No newline at end of file diff --git a/client/package.json b/client/package.json index 3d6434c4d..348e14508 100644 --- a/client/package.json +++ b/client/package.json @@ -58,6 +58,8 @@ "request-ip": "^3.3.0", "sass": "^1.58.3", "tunnel": "^0.0.6", + "winston": "^3.10.0", + "winston-mongodb": "^5.1.1", "zustand": "^4.3.5" }, "devDependencies": { diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index cf32885c2..12c458c71 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -152,6 +152,12 @@ dependencies: tunnel: specifier: ^0.0.6 version: registry.npmmirror.com/tunnel@0.0.6 + winston: + specifier: ^3.10.0 + version: registry.npmmirror.com/winston@3.10.0 + winston-mongodb: + specifier: ^5.1.1 + version: registry.npmmirror.com/winston-mongodb@5.1.1(winston@3.10.0) zustand: specifier: ^4.3.5 version: registry.npmmirror.com/zustand@4.3.5(immer@9.0.19)(react@18.2.0) @@ -4155,6 +4161,23 @@ packages: react: registry.npmmirror.com/react@18.2.0 dev: false + registry.npmmirror.com/@colors/colors@1.5.0: + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@colors/colors/-/colors-1.5.0.tgz} + name: '@colors/colors' + version: 1.5.0 + engines: {node: '>=0.1.90'} + dev: false + + registry.npmmirror.com/@dabh/diagnostics@2.0.3: + resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz} + name: '@dabh/diagnostics' + version: 2.0.3 + dependencies: + colorspace: registry.npmmirror.com/colorspace@1.1.4 + enabled: registry.npmmirror.com/enabled@2.0.0 + kuler: registry.npmmirror.com/kuler@2.0.0 + dev: false + registry.npmmirror.com/@dqbd/tiktoken@1.0.7: resolution: {integrity: sha512-bhR5k5W+8GLzysjk8zTMVygQZsgvf7W1F0IlL4ZQ5ugjo5rCyiwGM5d8DYriXspytfu98tv59niang3/T+FoDw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@dqbd/tiktoken/-/tiktoken-1.0.7.tgz} name: '@dqbd/tiktoken' @@ -5484,6 +5507,12 @@ packages: name: '@types/scheduler' version: 0.16.3 + registry.npmmirror.com/@types/triple-beam@1.3.2: + resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/triple-beam/-/triple-beam-1.3.2.tgz} + name: '@types/triple-beam' + version: 1.3.2 + dev: false + registry.npmmirror.com/@types/tunnel@0.0.3: resolution: {integrity: sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/tunnel/-/tunnel-0.0.3.tgz} name: '@types/tunnel' @@ -5771,6 +5800,12 @@ packages: version: 0.0.7 dev: true + registry.npmmirror.com/async@3.2.4: + resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/async/-/async-3.2.4.tgz} + name: async + version: 3.2.4 + dev: false + registry.npmmirror.com/asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz} name: asynckit @@ -5909,6 +5944,15 @@ packages: engines: {node: '>=8'} dev: false + registry.npmmirror.com/bl@2.2.1: + resolution: {integrity: sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bl/-/bl-2.2.1.tgz} + name: bl + version: 2.2.1 + dependencies: + readable-stream: registry.npmmirror.com/readable-stream@2.3.8 + safe-buffer: registry.npmmirror.com/safe-buffer@5.2.1 + dev: false + registry.npmmirror.com/bluebird@3.4.7: resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bluebird/-/bluebird-3.4.7.tgz} name: bluebird @@ -5966,6 +6010,13 @@ packages: node-releases: registry.npmmirror.com/node-releases@2.0.12 update-browserslist-db: registry.npmmirror.com/update-browserslist-db@1.0.11(browserslist@4.21.7) + registry.npmmirror.com/bson@1.1.6: + resolution: {integrity: sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bson/-/bson-1.1.6.tgz} + name: bson + version: 1.1.6 + engines: {node: '>=0.6.19'} + dev: false + registry.npmmirror.com/bson@4.7.2: resolution: {integrity: sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bson/-/bson-4.7.2.tgz} name: bson @@ -6136,7 +6187,15 @@ packages: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz} name: color-name version: 1.1.4 - dev: true + + registry.npmmirror.com/color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color-string/-/color-string-1.9.1.tgz} + name: color-string + version: 1.9.1 + dependencies: + color-name: registry.npmmirror.com/color-name@1.1.4 + simple-swizzle: registry.npmmirror.com/simple-swizzle@0.2.2 + dev: false registry.npmmirror.com/color2k@2.0.2: resolution: {integrity: sha512-kJhwH5nAwb34tmyuqq/lgjEKzlFXn1U99NlnB6Ws4qVaERcRUYeYP1cBw6BJ4vxaWStAUEef4WMr7WjOCnBt8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color2k/-/color2k-2.0.2.tgz} @@ -6144,6 +6203,24 @@ packages: version: 2.0.2 dev: false + registry.npmmirror.com/color@3.2.1: + resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color/-/color-3.2.1.tgz} + name: color + version: 3.2.1 + dependencies: + color-convert: registry.npmmirror.com/color-convert@1.9.3 + color-string: registry.npmmirror.com/color-string@1.9.1 + dev: false + + registry.npmmirror.com/colorspace@1.1.4: + resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/colorspace/-/colorspace-1.1.4.tgz} + name: colorspace + version: 1.1.4 + dependencies: + color: registry.npmmirror.com/color@3.2.1 + text-hex: registry.npmmirror.com/text-hex@1.0.0 + dev: false + registry.npmmirror.com/combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz} name: combined-stream @@ -6839,6 +6916,13 @@ packages: engines: {node: '>=0.4.0'} dev: false + registry.npmmirror.com/denque@1.5.1: + resolution: {integrity: sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/denque/-/denque-1.5.1.tgz} + name: denque + version: 1.5.1 + engines: {node: '>=0.10'} + dev: false + registry.npmmirror.com/dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dequal/-/dequal-2.0.3.tgz} name: dequal @@ -6984,6 +7068,12 @@ packages: version: 9.2.2 dev: true + registry.npmmirror.com/enabled@2.0.0: + resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/enabled/-/enabled-2.0.0.tgz} + name: enabled + version: 2.0.0 + dev: false + registry.npmmirror.com/enhanced-resolve@5.14.1: resolution: {integrity: sha512-Vklwq2vDKtl0y/vtwjSesgJ5MYS7Etuk5txS8VdKL4AOS1aUlD96zqIfsOSLQsdv3xgMRbtkWM8eG9XDfKUPow==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-5.14.1.tgz} name: enhanced-resolve @@ -7557,6 +7647,12 @@ packages: format: registry.npmmirror.com/format@0.2.2 dev: false + registry.npmmirror.com/fecha@4.2.3: + resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/fecha/-/fecha-4.2.3.tgz} + name: fecha + version: 4.2.3 + dev: false + registry.npmmirror.com/file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz} name: file-entry-cache @@ -7606,6 +7702,12 @@ packages: version: 3.2.7 dev: true + registry.npmmirror.com/fn.name@1.1.0: + resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/fn.name/-/fn.name-1.1.0.tgz} + name: fn.name + version: 1.1.0 + dev: false + registry.npmmirror.com/focus-lock@0.11.6: resolution: {integrity: sha512-KSuV3ur4gf2KqMNoZx3nXNVhqCkn42GuTYCX4tXPEwf0MjpFQmNMiN6m7dXaUXgIoivL6/65agoUMg4RLS0Vbg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/focus-lock/-/focus-lock-0.11.6.tgz} name: focus-lock @@ -8249,6 +8351,12 @@ packages: name: is-arrayish version: 0.2.1 + registry.npmmirror.com/is-arrayish@0.3.2: + resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-arrayish/-/is-arrayish-0.3.2.tgz} + name: is-arrayish + version: 0.3.2 + dev: false + registry.npmmirror.com/is-bigint@1.0.4: resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-bigint/-/is-bigint-1.0.4.tgz} name: is-bigint @@ -8429,7 +8537,6 @@ packages: name: is-stream version: 2.0.1 engines: {node: '>=8'} - dev: true registry.npmmirror.com/is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-stream/-/is-stream-3.0.0.tgz} @@ -8683,6 +8790,12 @@ packages: engines: {node: '>=6'} dev: false + registry.npmmirror.com/kuler@2.0.0: + resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/kuler/-/kuler-2.0.0.tgz} + name: kuler + version: 2.0.0 + dev: false + registry.npmmirror.com/language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz} name: language-subtag-registry @@ -8771,6 +8884,19 @@ packages: version: 4.17.21 dev: false + registry.npmmirror.com/logform@2.5.1: + resolution: {integrity: sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/logform/-/logform-2.5.1.tgz} + name: logform + version: 2.5.1 + dependencies: + '@colors/colors': registry.npmmirror.com/@colors/colors@1.5.0 + '@types/triple-beam': registry.npmmirror.com/@types/triple-beam@1.3.2 + fecha: registry.npmmirror.com/fecha@4.2.3 + ms: registry.npmmirror.com/ms@2.1.3 + safe-stable-stringify: registry.npmmirror.com/safe-stable-stringify@2.4.3 + triple-beam: registry.npmmirror.com/triple-beam@1.4.1 + dev: false + registry.npmmirror.com/longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/longest-streak/-/longest-streak-3.1.0.tgz} name: longest-streak @@ -9450,6 +9576,41 @@ packages: whatwg-url: registry.npmmirror.com/whatwg-url@11.0.0 dev: false + registry.npmmirror.com/mongodb@3.7.4: + resolution: {integrity: sha512-K5q8aBqEXMwWdVNh94UQTwZ6BejVbFhh1uB6c5FKtPE9eUMZPUO3sRZdgIEcHSrAWmxzpG/FeODDKL388sqRmw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mongodb/-/mongodb-3.7.4.tgz} + name: mongodb + version: 3.7.4 + engines: {node: '>=4'} + peerDependencies: + aws4: '*' + bson-ext: '*' + kerberos: '*' + mongodb-client-encryption: '*' + mongodb-extjson: '*' + snappy: '*' + peerDependenciesMeta: + aws4: + optional: true + bson-ext: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + mongodb-extjson: + optional: true + snappy: + optional: true + dependencies: + bl: registry.npmmirror.com/bl@2.2.1 + bson: registry.npmmirror.com/bson@1.1.6 + denque: registry.npmmirror.com/denque@1.5.1 + optional-require: registry.npmmirror.com/optional-require@1.1.8 + safe-buffer: registry.npmmirror.com/safe-buffer@5.2.1 + optionalDependencies: + saslprep: registry.npmmirror.com/saslprep@1.0.3 + dev: false + registry.npmmirror.com/mongodb@4.14.0: resolution: {integrity: sha512-coGKkWXIBczZPr284tYKFLg+KbGPPLlSbdgfKAb6QqCFt5bo5VFZ50O3FFzsw4rnkqjwT6D8Qcoo9nshYKM7Mg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mongodb/-/mongodb-4.14.0.tgz} name: mongodb @@ -9763,6 +9924,14 @@ packages: dependencies: wrappy: registry.npmmirror.com/wrappy@1.0.2 + registry.npmmirror.com/one-time@1.0.0: + resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/one-time/-/one-time-1.0.0.tgz} + name: one-time + version: 1.0.0 + dependencies: + fn.name: registry.npmmirror.com/fn.name@1.1.0 + dev: false + registry.npmmirror.com/onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz} name: onetime @@ -9810,6 +9979,15 @@ packages: version: 0.2.4 dev: false + registry.npmmirror.com/optional-require@1.1.8: + resolution: {integrity: sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/optional-require/-/optional-require-1.1.8.tgz} + name: optional-require + version: 1.1.8 + engines: {node: '>=4'} + dependencies: + require-at: registry.npmmirror.com/require-at@1.0.6 + dev: false + registry.npmmirror.com/optionator@0.9.1: resolution: {integrity: sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/optionator/-/optionator-0.9.1.tgz} name: optionator @@ -10392,6 +10570,17 @@ packages: util-deprecate: registry.npmmirror.com/util-deprecate@1.0.2 dev: false + registry.npmmirror.com/readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.2.tgz} + name: readable-stream + version: 3.6.2 + engines: {node: '>= 6'} + dependencies: + inherits: registry.npmmirror.com/inherits@2.0.4 + string_decoder: registry.npmmirror.com/string_decoder@1.1.1 + util-deprecate: registry.npmmirror.com/util-deprecate@1.0.2 + dev: false + registry.npmmirror.com/readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz} name: readdirp @@ -10569,6 +10758,13 @@ packages: version: 3.3.0 dev: false + registry.npmmirror.com/require-at@1.0.6: + resolution: {integrity: sha512-7i1auJbMUrXEAZCOQ0VNJgmcT2VOKPRl2YGJwgpHpC9CE91Mv4/4UYIUm4chGJaI381ZDq1JUicFii64Hapd8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/require-at/-/require-at-1.0.6.tgz} + name: require-at + version: 1.0.6 + engines: {node: '>=4'} + dev: false + registry.npmmirror.com/resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz} name: resolve-from @@ -10678,6 +10874,13 @@ packages: is-regex: registry.npmmirror.com/is-regex@1.1.4 dev: true + registry.npmmirror.com/safe-stable-stringify@2.4.3: + resolution: {integrity: sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz} + name: safe-stable-stringify + version: 2.4.3 + engines: {node: '>=10'} + dev: false + registry.npmmirror.com/safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz} name: safer-buffer @@ -10779,6 +10982,14 @@ packages: version: 3.0.7 dev: true + registry.npmmirror.com/simple-swizzle@0.2.2: + resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz} + name: simple-swizzle + version: 0.2.2 + dependencies: + is-arrayish: registry.npmmirror.com/is-arrayish@0.3.2 + dev: false + registry.npmmirror.com/slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz} name: slash @@ -10873,6 +11084,12 @@ packages: deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility' dev: true + registry.npmmirror.com/stack-trace@0.0.10: + resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/stack-trace/-/stack-trace-0.0.10.tgz} + name: stack-trace + version: 0.0.10 + dev: false + registry.npmmirror.com/stop-iteration-iterator@1.0.0: resolution: {integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz} name: stop-iteration-iterator @@ -11077,6 +11294,12 @@ packages: engines: {node: '>=6'} dev: true + registry.npmmirror.com/text-hex@1.0.0: + resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/text-hex/-/text-hex-1.0.0.tgz} + name: text-hex + version: 1.0.0 + dev: false + registry.npmmirror.com/text-table@0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/text-table/-/text-table-0.2.0.tgz} name: text-table @@ -11131,6 +11354,13 @@ packages: version: 3.0.1 dev: false + registry.npmmirror.com/triple-beam@1.4.1: + resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/triple-beam/-/triple-beam-1.4.1.tgz} + name: triple-beam + version: 1.4.1 + engines: {node: '>= 14.0.0'} + dev: false + registry.npmmirror.com/trough@2.1.0: resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/trough/-/trough-2.1.0.tgz} name: trough @@ -11589,6 +11819,57 @@ packages: isexe: registry.npmmirror.com/isexe@2.0.0 dev: true + registry.npmmirror.com/winston-mongodb@5.1.1(winston@3.10.0): + resolution: {integrity: sha512-tlDksYDCsSka6vLzB/cg19B+kRbVH/y9JQdXVXc5r4TSLEgX1Ivj/vHyedYlYzNMAZkSSfexwI/dSDn3f9/Qkg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/winston-mongodb/-/winston-mongodb-5.1.1.tgz} + id: registry.npmmirror.com/winston-mongodb/5.1.1 + name: winston-mongodb + version: 5.1.1 + engines: {node: '>=6.8.1'} + peerDependencies: + winston: ^3.0.0 + dependencies: + mongodb: registry.npmmirror.com/mongodb@3.7.4 + winston: registry.npmmirror.com/winston@3.10.0 + winston-transport: registry.npmmirror.com/winston-transport@4.5.0 + transitivePeerDependencies: + - aws4 + - bson-ext + - kerberos + - mongodb-client-encryption + - mongodb-extjson + - snappy + dev: false + + registry.npmmirror.com/winston-transport@4.5.0: + resolution: {integrity: sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/winston-transport/-/winston-transport-4.5.0.tgz} + name: winston-transport + version: 4.5.0 + engines: {node: '>= 6.4.0'} + dependencies: + logform: registry.npmmirror.com/logform@2.5.1 + readable-stream: registry.npmmirror.com/readable-stream@3.6.2 + triple-beam: registry.npmmirror.com/triple-beam@1.4.1 + dev: false + + registry.npmmirror.com/winston@3.10.0: + resolution: {integrity: sha512-nT6SIDaE9B7ZRO0u3UvdrimG0HkB7dSTAgInQnNR2SOPJ4bvq5q79+pXLftKmP52lJGW15+H5MCK0nM9D3KB/g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/winston/-/winston-3.10.0.tgz} + name: winston + version: 3.10.0 + engines: {node: '>= 12.0.0'} + dependencies: + '@colors/colors': registry.npmmirror.com/@colors/colors@1.5.0 + '@dabh/diagnostics': registry.npmmirror.com/@dabh/diagnostics@2.0.3 + async: registry.npmmirror.com/async@3.2.4 + is-stream: registry.npmmirror.com/is-stream@2.0.1 + logform: registry.npmmirror.com/logform@2.5.1 + one-time: registry.npmmirror.com/one-time@1.0.0 + readable-stream: registry.npmmirror.com/readable-stream@3.6.2 + safe-stable-stringify: registry.npmmirror.com/safe-stable-stringify@2.4.3 + stack-trace: registry.npmmirror.com/stack-trace@0.0.10 + triple-beam: registry.npmmirror.com/triple-beam@1.4.1 + winston-transport: registry.npmmirror.com/winston-transport@4.5.0 + dev: false + registry.npmmirror.com/word-wrap@1.2.3: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/word-wrap/-/word-wrap-1.2.3.tgz} name: word-wrap diff --git a/client/src/constants/flow/ModuleTemplate.ts b/client/src/constants/flow/ModuleTemplate.ts index 09f0341f6..874ab87ea 100644 --- a/client/src/constants/flow/ModuleTemplate.ts +++ b/client/src/constants/flow/ModuleTemplate.ts @@ -182,8 +182,9 @@ export const ChatModule: FlowModuleTemplateType = { { key: TaskResponseKeyEnum.answerText, label: '模型回复', - description: '直接响应，无需配置', - type: FlowOutputItemTypeEnum.hidden, + description: '如果外接了内容，会在回复结束时自动添加\n\n', + valueType: FlowValueTypeEnum.string, + type: FlowOutputItemTypeEnum.source, targets: [] }, { @@ -285,7 +286,16 @@ export const AnswerModule: FlowModuleTemplateType = { '可以使用 \\n 来实现换行。也可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容' } ], - outputs: [] + outputs: [ + { + key: 'finish', + label: '回复结束', + description: '回复完成后触发', + valueType: FlowValueTypeEnum.boolean, + type: FlowOutputItemTypeEnum.source, + targets: [] + } + ] }; export const TFSwitchModule: FlowModuleTemplateType = { logo: '', diff --git a/client/src/pages/api/openapi/v1/chat/completions.ts b/client/src/pages/api/openapi/v1/chat/completions.ts index f5b9d8f6b..e453bfaf6 100644 --- a/client/src/pages/api/openapi/v1/chat/completions.ts +++ b/client/src/pages/api/openapi/v1/chat/completions.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { connectToDatabase } from '@/service/mongo'; import { authUser, authApp, authShareChat, AuthUserTypeEnum } from '@/service/utils/auth'; import { sseErrRes, jsonRes } from '@/service/response'; -import { withNextCors } from '@/service/utils/tools'; +import { addLog, withNextCors } from '@/service/utils/tools'; import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat'; import { dispatchHistory, @@ -181,7 +181,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex }); } - console.log(`finish time: ${(Date.now() - startTime) / 1000}s`); + addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`); if (stream) { sseResponse({ @@ -351,6 +351,7 @@ export async function dispatchModules({ res, stream, detail, + outputs: module.outputs, userOpenaiAccount: user?.openaiAccount, ...params }; diff --git a/client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeAnswer.tsx b/client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeAnswer.tsx index eeb0f5ea9..1ff25f15d 100644 --- a/client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeAnswer.tsx +++ b/client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeAnswer.tsx @@ -4,6 +4,7 @@ import NodeCard from '../modules/NodeCard'; import { FlowModuleItemType } from '@/types/flow'; import Container from '../modules/Container'; import RenderInput from '../render/RenderInput'; +import RenderOutput from '../render/RenderOutput'; const NodeAnswer = ({ data }: NodeProps) => { const { moduleId, inputs, outputs, onChangeNode } = data; @@ -11,6 +12,7 @@ const NodeAnswer = ({ data }: NodeProps) => { + ); diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts index af1a5ae35..98adddfcc 100644 --- a/client/src/service/events/generateQA.ts +++ b/client/src/service/events/generateQA.ts @@ -43,7 +43,7 @@ export async function generateQA(): Promise { // task preemption if (!data) { reduceQueue(); - global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`); + global.qaQueueLen <= 0 && console.log(`【QA】任务完成`); return; } diff --git a/client/src/service/events/generateVector.ts b/client/src/service/events/generateVector.ts index 2d5514338..ec7b8ea42 100644 --- a/client/src/service/events/generateVector.ts +++ b/client/src/service/events/generateVector.ts @@ -44,7 +44,7 @@ export async function generateVector(): Promise { // task preemption if (!data) { reduceQueue(); - global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`); + global.vectorQueueLen <= 0 && console.log(`【索引】任务完成`); return; } diff --git a/client/src/service/events/pushBill.ts b/client/src/service/events/pushBill.ts index e8680acf9..63bfdc8b1 100644 --- a/client/src/service/events/pushBill.ts +++ b/client/src/service/events/pushBill.ts @@ -3,6 +3,7 @@ import { BillSourceEnum } from '@/constants/user'; import { getModel } from '../utils/data'; import { ChatHistoryItemResType } from '@/types/chat'; import { formatPrice } from '@/utils/user'; +import { addLog } from '../utils/tools'; export const pushTaskBill = async ({ appName, @@ -48,7 +49,11 @@ export const pushTaskBill = async ({ : []) ]); - console.log('finish bill:', formatPrice(total)); + addLog.info(`finish completions`, { + source, + userId, + price: formatPrice(total) + }); }; export const updateShareChatBill = async ({ @@ -66,8 +71,8 @@ export const updateShareChatBill = async ({ lastTime: new Date() } ); - } catch (error) { - console.log('update shareChat error', error); + } catch (err) { + addLog.error('update shareChat error', { err }); } }; @@ -82,7 +87,7 @@ export const pushSplitDataBill = async ({ totalTokens: number; appName: string; }) => { - console.log(`splitData generate success. token len: ${totalTokens}.`); + addLog.info('splitData generate success', { totalTokens }); let billId; @@ -107,8 +112,8 @@ export const pushSplitDataBill = async ({ await User.findByIdAndUpdate(userId, { $inc: { balance: -total } }); - } catch (error) { - console.log('创建账单失败:', error); + } catch (err) { + addLog.error('Create completions bill error', { err }); billId && Bill.findByIdAndDelete(billId); } }; @@ -156,8 +161,8 @@ export const pushGenerateVectorBill = async ({ await User.findByIdAndUpdate(userId, { $inc: { balance: -total } }); - } catch (error) { - console.log('创建账单失败:', error); + } catch (err) { + addLog.error('Create generateVector bill error', { err }); billId && Bill.findByIdAndDelete(billId); } } catch (error) { diff --git a/client/src/service/moduleDispatch/chat/oneapi.ts b/client/src/service/moduleDispatch/chat/oneapi.ts index 4caa88d44..2d515fc02 100644 --- a/client/src/service/moduleDispatch/chat/oneapi.ts +++ b/client/src/service/moduleDispatch/chat/oneapi.ts @@ -17,6 +17,7 @@ import { ChatModelItemType } from '@/types/model'; import { UserModelSchema } from '@/types/mongoSchema'; import { textCensor } from '@/service/api/plugins'; import { ChatCompletionRequestMessageRoleEnum } from 'openai'; +import { AppModuleItemType } from '@/types/app'; export type ChatProps = { res: NextApiResponse; @@ -31,6 +32,7 @@ export type ChatProps = { systemPrompt?: string; limitPrompt?: string; userOpenaiAccount: UserModelSchema['openaiAccount']; + outputs: AppModuleItemType['outputs']; }; export type ChatResponse = { [TaskResponseKeyEnum.answerText]: string; @@ -52,8 +54,12 @@ export const dispatchChatCompletion = async (props: Record): Promis userChatInput, systemPrompt = '', limitPrompt = '', - userOpenaiAccount + userOpenaiAccount, + outputs } = props as ChatProps; + if (!userChatInput) { + return Promise.reject('Question is empty'); + } // temperature adapt const modelConstantsData = getChatModel(model); @@ -142,6 +148,8 @@ export const dispatchChatCompletion = async (props: Record): Promis messages: completeMessages }); + targetResponse({ res, detail, outputs }); + return { answerText: answer, totalTokens, @@ -304,6 +312,28 @@ function getMaxTokens({ }; } +function targetResponse({ + res, + outputs, + detail +}: { + res: NextApiResponse; + outputs: AppModuleItemType['outputs']; + detail: boolean; +}) { + const targets = + outputs.find((output) => output.key === TaskResponseKeyEnum.answerText)?.targets || []; + + if (targets.length === 0) return; + sseResponse({ + res, + event: detail ? sseResponseEventEnum.answer : undefined, + data: textAdaptGptResponse({ + text: '\n' + }) + }); +} + async function streamResponse({ res, detail, diff --git a/client/src/service/moduleDispatch/tools/answer.ts b/client/src/service/moduleDispatch/tools/answer.ts index b061a0134..dc11bf38f 100644 --- a/client/src/service/moduleDispatch/tools/answer.ts +++ b/client/src/service/moduleDispatch/tools/answer.ts @@ -11,6 +11,7 @@ export type AnswerProps = { }; export type AnswerResponse = { [TaskResponseKeyEnum.answerText]: string; + finish: boolean; }; export const dispatchAnswer = (props: Record): AnswerResponse => { @@ -27,6 +28,7 @@ export const dispatchAnswer = (props: Record): AnswerResponse => { } return { - [TaskResponseKeyEnum.answerText]: text + [TaskResponseKeyEnum.answerText]: text, + finish: true }; }; diff --git a/client/src/service/mongo.ts b/client/src/service/mongo.ts index 8e7a538ae..5ac3bcbc7 100644 --- a/client/src/service/mongo.ts +++ b/client/src/service/mongo.ts @@ -7,6 +7,8 @@ import { PRICE_SCALE } from '@/constants/common'; import { connectPg, PgClient } from './pg'; import { createHashPassword } from '@/utils/tools'; import { PgTrainingTableName } from '@/constants/plugin'; +import { createLogger, format, transports } from 'winston'; +import 'winston-mongodb'; /** * connect MongoDB and init data @@ -32,6 +34,9 @@ export async function connectToDatabase(): Promise { }); } + // logger + initLogger(); + // init function getInitConfig(); @@ -39,7 +44,6 @@ export async function connectToDatabase(): Promise { mongoose.set('strictQuery', true); global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, { bufferCommands: true, - dbName: process.env.MONGODB_NAME, maxConnecting: Number(process.env.DB_MAX_LINK || 5), maxPoolSize: Number(process.env.DB_MAX_LINK || 5), minPoolSize: 2 @@ -57,6 +61,37 @@ export async function connectToDatabase(): Promise { startQueue(); } +function initLogger() { + global.logger = createLogger({ + transports: [ + new transports.MongoDB({ + db: process.env.MONGODB_URI as string, + collection: 'server_logs', + options: { + useUnifiedTopology: true + }, + cappedSize: 500000000, + tryReconnect: true, + metaKey: 'meta', + format: format.combine(format.timestamp(), format.json()) + }), + new transports.Console({ + format: format.combine( + format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + format.printf((info) => { + if (info.level === 'error') { + console.log(info.meta); + return `${info.level}: ${[info.timestamp]}: ${info.message}`; + } + return `${info.level}: ${[info.timestamp]}: ${info.message}${ + info.meta ? `: ${JSON.stringify(info.meta)}` : '' + }`; + }) + ) + }) + ] + }); +} async function initRootUser() { try { const rootUser = await User.findOne({ diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index 65452976a..239378c06 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -8,13 +8,9 @@ export const connectPg = async () => { } global.pgClient = new Pool({ - host: process.env.PG_HOST, - port: process.env.PG_PORT ? +process.env.PG_PORT : 5432, - user: process.env.PG_USER, - password: process.env.PG_PASSWORD, - database: process.env.PG_DB_NAME, + connectionString: process.env.PG_URL, max: Number(process.env.DB_MAX_LINK || 5), - idleTimeoutMillis: 30000, + keepAlive: true, connectionTimeoutMillis: 5000 }); diff --git a/client/src/service/response.ts b/client/src/service/response.ts index 576486f9d..ba8680a29 100644 --- a/client/src/service/response.ts +++ b/client/src/service/response.ts @@ -7,7 +7,7 @@ import { ERROR_RESPONSE, ERROR_ENUM } from './errorCode'; -import { clearCookie, sseResponse } from './utils/tools'; +import { clearCookie, sseResponse, addLog } from './utils/tools'; export interface ResponseType { code: number; @@ -52,7 +52,24 @@ export const jsonRes = ( } else if (openaiError[error?.response?.statusText]) { msg = openaiError[error.response.statusText]; } - console.log(error); + + addLog.error(msg, { + message: error.message, + stack: error.stack, + ...(error.config && { + config: { + headers: error.config.headers, + url: error.config.url, + data: error.config.data + } + }), + ...(error.response && { + response: { + status: error.response.status, + statusText: error.response.statusText + } + }) + }); } res.status(code).json({ @@ -92,7 +109,24 @@ export const sseErrRes = (res: NextApiResponse, error: any) => { } else if (openaiError[error?.response?.statusText]) { msg = openaiError[error.response.statusText]; } - console.log('sse error => ', error); + + addLog.error(`sse error: ${msg}`, { + message: error.message, + stack: error.stack, + ...(error.config && { + config: { + headers: error.config.headers, + url: error.config.url, + data: error.config.data + } + }), + ...(error.response && { + response: { + status: error.response.status, + statusText: error.response.statusText + } + }) + }); sseResponse({ res, diff --git a/client/src/service/utils/tools.ts b/client/src/service/utils/tools.ts index 959647c21..5f2240912 100644 --- a/client/src/service/utils/tools.ts +++ b/client/src/service/utils/tools.ts @@ -65,6 +65,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler { }; } +/* start task */ export const startQueue = () => { for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) { generateQA(); @@ -87,3 +88,13 @@ export const sseResponse = ({ event && res.write(`event: ${event}\n`); res.write(`data: ${data}\n\n`); }; + +/* add logger */ +export const addLog = { + info: (msg: string, obj?: Record) => { + global.logger.info(msg, { meta: obj }); + }, + error: (msg: string, obj?: Record) => { + global.logger.error(msg, { meta: obj }); + } +}; diff --git a/client/src/types/index.d.ts b/client/src/types/index.d.ts index 3e930e04f..a05f126e8 100644 --- a/client/src/types/index.d.ts +++ b/client/src/types/index.d.ts @@ -2,6 +2,7 @@ import type { Mongoose } from 'mongoose'; import type { Agent } from 'http'; import type { Pool } from 'pg'; import type { Tiktoken } from '@dqbd/tiktoken'; +import type { Logger } from 'winston'; import { ChatModelItemType, QAModelItemType, VectorModelItemType } from './model'; export type PagingData = { @@ -55,6 +56,9 @@ declare global { var qaQueueLen: number; var vectorQueueLen: number; var OpenAiEncMap: Tiktoken; + + var logger: Logger; + var sendInformQueue: (() => Promise)[]; var sendInformQueueLen: number; diff --git a/docSite/docs/develop/deploy/docker.md b/docSite/docs/develop/deploy/docker.md index 08e01693f..127e42351 100644 --- a/docSite/docs/develop/deploy/docker.md +++ b/docSite/docs/develop/deploy/docker.md @@ -89,15 +89,9 @@ services: - TOKEN_KEY=any - ROOT_KEY=root_key # mongo 配置，不需要改 - - MONGODB_URI=mongodb://username:password@mongo:27017 - # - MONGODB_URI=mongodb://username:password@mongo:27017/?authSource=admin - - MONGODB_NAME=fastgpt - # pg配置. - - PG_HOST=pg - - PG_PORT=5432 - - PG_USER=username - - PG_PASSWORD=password - - PG_DB_NAME=postgres + - MONGODB_URI=mongodb://username:password@mongo:27017/fastgpt + # pg配置. 不需要改 + - PG_URL=postgresql://username:password@pg:5432/postgres networks: fastgpt: ``` @@ -149,15 +143,9 @@ services: # root key, 最高权限，可以内部接口互相调用 - ROOT_KEY=root_key # mongo 配置，不需要改 - - MONGODB_URI=mongodb://username:password@0.0.0.0:27017 - # - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin - - MONGODB_NAME=fastgpt - # pg 配置 - - PG_HOST=0.0.0.0 - - PG_PORT=5432 - - PG_USER=username - - PG_PASSWORD=password - - PG_DB_NAME=postgres + - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt + # pg配置. 不需要改 + - PG_URL=postgresql://username:password@0.0.0.0:5432/postgres ``` ## 四、运行 docker-compose diff --git a/docSite/docs/develop/deploy/v4init.md b/docSite/docs/develop/update/40init.md similarity index 98% rename from docSite/docs/develop/deploy/v4init.md rename to docSite/docs/develop/update/40init.md index 14bc131a7..d986d575d 100644 --- a/docSite/docs/develop/deploy/v4init.md +++ b/docSite/docs/develop/update/40init.md @@ -1,4 +1,4 @@ -# V4 版本初始化 +# V4.0 版本初始化 新版 mongo 表进行了不少的变更，需要执行一些初始化脚本。 diff --git a/docSite/docs/develop/deploy/initv4.1.md b/docSite/docs/develop/update/41init.md similarity index 55% rename from docSite/docs/develop/deploy/initv4.1.md rename to docSite/docs/develop/update/41init.md index e64a94b2f..ecf1ca1fd 100644 --- a/docSite/docs/develop/deploy/initv4.1.md +++ b/docSite/docs/develop/update/41init.md @@ -2,6 +2,15 @@ 新版重新设置了对话存储结构，需要初始化原来的存储内容 +## 更新环境变量 + +优化了 PG 和 Mongo 的连接变量，只需要 1 个 url 即可。 + +``` +MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt +PG_URL=postgresql://username:password@0.0.0.0:5432/postgres +``` + ## 执行初始化 API 部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） diff --git a/docSite/docusaurus.config.js b/docSite/docusaurus.config.js index 2cbf23a6f..1ed85f40a 100644 --- a/docSite/docusaurus.config.js +++ b/docSite/docusaurus.config.js @@ -28,8 +28,7 @@ const config = { ({ docs: { sidebarPath: require.resolve('./sidebars.js'), - editUrl: - 'https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/' + editUrl: 'https://github.com/labring/FastGPT/blob/main/docSite/' }, theme: { customCss: require.resolve('./src/css/custom.css') diff --git a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current.json b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current.json index e4a9e6ec5..23d3c91e5 100644 --- a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current.json +++ b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current.json @@ -14,6 +14,9 @@ "sidebar.docSidebar.category.Deploy": { "message": "部署" }, + "sidebar.docSidebar.category.Version Updating": { + "message": "版本更新" + }, "sidebar.docSidebar.category.Datasets": { "message": "知识库实践" }, diff --git a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/docker.md b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/docker.md index dde785f07..1d5595775 100644 --- a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/docker.md +++ b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/docker.md @@ -89,15 +89,9 @@ services: - TOKEN_KEY=any - ROOT_KEY=root_key # mongo 配置，不需要改 - - MONGODB_URI=mongodb://username:password@mongo:27017 # 如果这个连不上，尝试下面的 - # - MONGODB_URI=mongodb://username:password@mongo:27017/?authSource=admin - - MONGODB_NAME=fastgpt - # pg配置. - - PG_HOST=pg - - PG_PORT=5432 - - PG_USER=username - - PG_PASSWORD=password - - PG_DB_NAME=postgres + - MONGODB_URI=mongodb://username:password@mongo:27017/fastgpt + # pg配置. 不需要改 + - PG_URL=postgresql://username:password@pg:5432/postgres networks: fastgpt: ``` @@ -153,15 +147,9 @@ environment: # root key, 最高权限，可以内部接口互相调用 - ROOT_KEY=root_key # mongo 配置，不需要改 - - MONGODB_URI=mongodb://username:password@0.0.0.0:27017 - # - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin - - MONGODB_NAME=fastgpt - # pg 配置 - - PG_HOST=0.0.0.0 - - PG_PORT=5432 - - PG_USER=username - - PG_PASSWORD=password - - PG_DB_NAME=postgres + - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt + # pg配置. 不需要改 + - PG_URL=postgresql://username:password@0.0.0.0:5432/postgres ``` ## 四、运行 docker-compose diff --git a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/v4init.md b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/40init.md similarity index 98% rename from docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/v4init.md rename to docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/40init.md index 14bc131a7..d986d575d 100644 --- a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/v4init.md +++ b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/40init.md @@ -1,4 +1,4 @@ -# V4 版本初始化 +# V4.0 版本初始化 新版 mongo 表进行了不少的变更，需要执行一些初始化脚本。 diff --git a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/initv4.1.md b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md similarity index 55% rename from docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/initv4.1.md rename to docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md index e64a94b2f..ecf1ca1fd 100644 --- a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/deploy/initv4.1.md +++ b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md @@ -2,6 +2,15 @@ 新版重新设置了对话存储结构，需要初始化原来的存储内容 +## 更新环境变量 + +优化了 PG 和 Mongo 的连接变量，只需要 1 个 url 即可。 + +``` +MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt +PG_URL=postgresql://username:password@0.0.0.0:5432/postgres +``` + ## 执行初始化 API 部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） diff --git a/docSite/sidebars.js b/docSite/sidebars.js index 69bf38d7d..75c603aa5 100644 --- a/docSite/sidebars.js +++ b/docSite/sidebars.js @@ -53,7 +53,20 @@ const sidebars = { } ] }, - 'develop/oneapi' + 'develop/oneapi', + { + type: 'category', + label: 'Version Updating', + link: { + type: 'generated-index' + }, + items: [ + { + type: 'autogenerated', + dirName: 'develop/update' + } + ] + } ] }, { diff --git a/files/deploy/fastgpt/docker-compose.yml b/files/deploy/fastgpt/docker-compose.yml index 43a7b8657..29555763b 100644 --- a/files/deploy/fastgpt/docker-compose.yml +++ b/files/deploy/fastgpt/docker-compose.yml @@ -54,14 +54,9 @@ services: - TOKEN_KEY=any - ROOT_KEY=root_key # mongo 配置，不需要改 - - MONGODB_URI=mongodb://username:password@mongo:27017/?authSource=admin - - MONGODB_NAME=fastgpt - # pg配置. - - PG_HOST=pg - - PG_PORT=5432 - - PG_USER=username - - PG_PASSWORD=password - - PG_DB_NAME=postgres + - MONGODB_URI=mongodb://username:password@mongo:27017/fastgpt + # pg配置. 不需要改 + - PG_URL=postgresql://username:password@pg:5432/postgres volumes: - ./config.json:/app/data/config.json networks: @@ -116,11 +111,6 @@ networks: # # root key, 最高权限，可以内部接口互相调用 # - ROOT_KEY=root_key # # mongo 配置，不需要改 -# - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin -# - MONGODB_NAME=fastgpt -# # pg 配置 -# - PG_HOST=0.0.0.0 -# - PG_PORT=5432 -# - PG_USER=username -# - PG_PASSWORD=password -# - PG_DB_NAME=postgres +# - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt +# # pg配置. 不需要改 +# - PG_URL=postgresql://username:password@0.0.0.0:5432/postgres