Skip to content

Commit 929a851

Browse files
xbingWawesomeYGdhsifss
authored
Feat/v2.45.0 (#246)
* refactor: remove unused MenuItem for viewing folder documents in KnowledgeBasePage - Eliminated the MenuItem component for viewing documents in the current folder, streamlining the UI. - Retained the refresh functionality for the folder. * feat: add reindexing functionality for documents in Knowledge Base - Introduced a new API endpoint for reindexing documents, allowing for improved document learning. - Implemented UI components for batch reindexing and individual document reindexing across various pages. - Enhanced user experience with contextual menus for document actions, including reindexing and deletion. - Updated relevant types to support new functionality. * feat: doc reindex * chore: update raglite * fix: file type filter * feat: add reindexing functionality for space folders in Knowledge Base - Introduced a new API endpoint for reindexing space folders, enhancing document learning capabilities. - Updated UI components to support reindexing actions, including contextual menus for folder operations. - Implemented validation to prevent reindexing of documents in pending review status. - Enhanced user experience with appropriate warning messages for unsupported actions. * fix: folder reindex --------- Co-authored-by: awesomeYG <gang.yang@chaitin.com> Co-authored-by: 姚凯 <kai.yao@chaitin.com>
1 parent abd3ce3 commit 929a851

25 files changed

Lines changed: 842 additions & 159 deletions

File tree

‎.github/workflows/build-base-img.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
- name: raglite
4545
dockerfile: docker/raglite/Dockerfile
4646
context: docker/raglite
47-
tag: "v2.15.3"
47+
tag: "v2.16.0"
4848

4949
steps:
5050
- name: Checkout code

‎backend/docs/docs.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1550,6 +1550,46 @@ const docTemplate = `{
15501550
}
15511551
}
15521552
},
1553+
"/admin/kb/{kb_id}/document/reindex": {
1554+
"put": {
1555+
"consumes": [
1556+
"application/json"
1557+
],
1558+
"produces": [
1559+
"application/json"
1560+
],
1561+
"tags": [
1562+
"document"
1563+
],
1564+
"summary": "reindex doc",
1565+
"parameters": [
1566+
{
1567+
"type": "integer",
1568+
"description": "kb_id",
1569+
"name": "kb_id",
1570+
"in": "path",
1571+
"required": true
1572+
},
1573+
{
1574+
"description": "request params",
1575+
"name": "req",
1576+
"in": "body",
1577+
"required": true,
1578+
"schema": {
1579+
"$ref": "#/definitions/svc.DocReindexReq"
1580+
}
1581+
}
1582+
],
1583+
"responses": {
1584+
"200": {
1585+
"description": "OK",
1586+
"schema": {
1587+
"$ref": "#/definitions/context.Response"
1588+
}
1589+
}
1590+
}
1591+
}
1592+
},
15531593
"/admin/kb/{kb_id}/document/{doc_id}": {
15541594
"get": {
15551595
"produces": [
@@ -2620,6 +2660,48 @@ const docTemplate = `{
26202660
}
26212661
}
26222662
},
2663+
"/admin/kb/{kb_id}/space/{space_id}/folder/{folder_id}/reindex": {
2664+
"put": {
2665+
"produces": [
2666+
"application/json"
2667+
],
2668+
"tags": [
2669+
"space"
2670+
],
2671+
"summary": "reindex kb space folder",
2672+
"parameters": [
2673+
{
2674+
"type": "integer",
2675+
"description": "kb_id",
2676+
"name": "kb_id",
2677+
"in": "path",
2678+
"required": true
2679+
},
2680+
{
2681+
"type": "integer",
2682+
"description": "space_id",
2683+
"name": "space_id",
2684+
"in": "path",
2685+
"required": true
2686+
},
2687+
{
2688+
"type": "integer",
2689+
"description": "folder_id",
2690+
"name": "folder_id",
2691+
"in": "path",
2692+
"required": true
2693+
}
2694+
],
2695+
"responses": {
2696+
"200": {
2697+
"description": "OK",
2698+
"schema": {
2699+
"$ref": "#/definitions/context.Response"
2700+
}
2701+
}
2702+
}
2703+
}
2704+
},
26232705
"/admin/kb/{kb_id}/space/{space_id}/refresh": {
26242706
"put": {
26252707
"produces": [
@@ -11244,6 +11326,17 @@ const docTemplate = `{
1124411326
}
1124511327
}
1124611328
},
11329+
"svc.DocReindexReq": {
11330+
"type": "object",
11331+
"properties": {
11332+
"ids": {
11333+
"type": "array",
11334+
"items": {
11335+
"type": "integer"
11336+
}
11337+
}
11338+
}
11339+
},
1124711340
"svc.DocUpdateReq": {
1124811341
"type": "object",
1124911342
"required": [

‎backend/docs/swagger.json‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1543,6 +1543,46 @@
15431543
}
15441544
}
15451545
},
1546+
"/admin/kb/{kb_id}/document/reindex": {
1547+
"put": {
1548+
"consumes": [
1549+
"application/json"
1550+
],
1551+
"produces": [
1552+
"application/json"
1553+
],
1554+
"tags": [
1555+
"document"
1556+
],
1557+
"summary": "reindex doc",
1558+
"parameters": [
1559+
{
1560+
"type": "integer",
1561+
"description": "kb_id",
1562+
"name": "kb_id",
1563+
"in": "path",
1564+
"required": true
1565+
},
1566+
{
1567+
"description": "request params",
1568+
"name": "req",
1569+
"in": "body",
1570+
"required": true,
1571+
"schema": {
1572+
"$ref": "#/definitions/svc.DocReindexReq"
1573+
}
1574+
}
1575+
],
1576+
"responses": {
1577+
"200": {
1578+
"description": "OK",
1579+
"schema": {
1580+
"$ref": "#/definitions/context.Response"
1581+
}
1582+
}
1583+
}
1584+
}
1585+
},
15461586
"/admin/kb/{kb_id}/document/{doc_id}": {
15471587
"get": {
15481588
"produces": [
@@ -2613,6 +2653,48 @@
26132653
}
26142654
}
26152655
},
2656+
"/admin/kb/{kb_id}/space/{space_id}/folder/{folder_id}/reindex": {
2657+
"put": {
2658+
"produces": [
2659+
"application/json"
2660+
],
2661+
"tags": [
2662+
"space"
2663+
],
2664+
"summary": "reindex kb space folder",
2665+
"parameters": [
2666+
{
2667+
"type": "integer",
2668+
"description": "kb_id",
2669+
"name": "kb_id",
2670+
"in": "path",
2671+
"required": true
2672+
},
2673+
{
2674+
"type": "integer",
2675+
"description": "space_id",
2676+
"name": "space_id",
2677+
"in": "path",
2678+
"required": true
2679+
},
2680+
{
2681+
"type": "integer",
2682+
"description": "folder_id",
2683+
"name": "folder_id",
2684+
"in": "path",
2685+
"required": true
2686+
}
2687+
],
2688+
"responses": {
2689+
"200": {
2690+
"description": "OK",
2691+
"schema": {
2692+
"$ref": "#/definitions/context.Response"
2693+
}
2694+
}
2695+
}
2696+
}
2697+
},
26162698
"/admin/kb/{kb_id}/space/{space_id}/refresh": {
26172699
"put": {
26182700
"produces": [
@@ -11237,6 +11319,17 @@
1123711319
}
1123811320
}
1123911321
},
11322+
"svc.DocReindexReq": {
11323+
"type": "object",
11324+
"properties": {
11325+
"ids": {
11326+
"type": "array",
11327+
"items": {
11328+
"type": "integer"
11329+
}
11330+
}
11331+
}
11332+
},
1124011333
"svc.DocUpdateReq": {
1124111334
"type": "object",
1124211335
"required": [

‎backend/docs/swagger.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1860,6 +1860,13 @@ definitions:
18601860
updated_at:
18611861
type: integer
18621862
type: object
1863+
svc.DocReindexReq:
1864+
properties:
1865+
ids:
1866+
items:
1867+
type: integer
1868+
type: array
1869+
type: object
18631870
svc.DocUpdateReq:
18641871
properties:
18651872
desc:
@@ -3449,6 +3456,32 @@ paths:
34493456
summary: update doc group_ids
34503457
tags:
34513458
- document
3459+
/admin/kb/{kb_id}/document/reindex:
3460+
put:
3461+
consumes:
3462+
- application/json
3463+
parameters:
3464+
- description: kb_id
3465+
in: path
3466+
name: kb_id
3467+
required: true
3468+
type: integer
3469+
- description: request params
3470+
in: body
3471+
name: req
3472+
required: true
3473+
schema:
3474+
$ref: '#/definitions/svc.DocReindexReq'
3475+
produces:
3476+
- application/json
3477+
responses:
3478+
"200":
3479+
description: OK
3480+
schema:
3481+
$ref: '#/definitions/context.Response'
3482+
summary: reindex doc
3483+
tags:
3484+
- document
34523485
/admin/kb/{kb_id}/question:
34533486
get:
34543487
parameters:
@@ -4089,6 +4122,34 @@ paths:
40894122
summary: list kb space folder doc
40904123
tags:
40914124
- space
4125+
/admin/kb/{kb_id}/space/{space_id}/folder/{folder_id}/reindex:
4126+
put:
4127+
parameters:
4128+
- description: kb_id
4129+
in: path
4130+
name: kb_id
4131+
required: true
4132+
type: integer
4133+
- description: space_id
4134+
in: path
4135+
name: space_id
4136+
required: true
4137+
type: integer
4138+
- description: folder_id
4139+
in: path
4140+
name: folder_id
4141+
required: true
4142+
type: integer
4143+
produces:
4144+
- application/json
4145+
responses:
4146+
"200":
4147+
description: OK
4148+
schema:
4149+
$ref: '#/definitions/context.Response'
4150+
summary: reindex kb space folder
4151+
tags:
4152+
- space
40924153
/admin/kb/{kb_id}/space/{space_id}/refresh:
40934154
put:
40944155
parameters:

‎backend/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ require (
99
github.com/alibabacloud-go/tea-utils/v2 v2.0.7
1010
github.com/caarlos0/env/v11 v11.3.1
1111
github.com/chaitin/ModelKit/v2 v2.9.1
12-
github.com/chaitin/raglite-go-sdk v0.2.1
12+
github.com/chaitin/raglite-go-sdk v0.2.2
1313
github.com/cloudwego/eino v0.7.3
1414
github.com/coreos/go-oidc/v3 v3.15.0
1515
github.com/gin-contrib/sessions v0.0.0-20190101140330-dc5246754963

‎backend/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
106106
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
107107
github.com/chaitin/ModelKit/v2 v2.9.1 h1:5Ns6dtJtXvVN1RDX4GfnP5xWnt1zcU+d/8vLMnRGspk=
108108
github.com/chaitin/ModelKit/v2 v2.9.1/go.mod h1:JgCZZlTCwNL+9aGbUFU9gkPYAEp32IJnTWEo+iIM/wk=
109-
github.com/chaitin/raglite-go-sdk v0.2.1 h1:iginJquZb9fy3Z2sK4g7uSdra73twK7oVVOeHKB5WUU=
110-
github.com/chaitin/raglite-go-sdk v0.2.1/go.mod h1:1klR7WqfFijmd4msUvhRHoGstteUfBsRuRdX4CIJ/so=
109+
github.com/chaitin/raglite-go-sdk v0.2.2 h1:b96EmNMrKaJ9HUwtmuxWGfaB7spTfJ69LxCjrZklJCU=
110+
github.com/chaitin/raglite-go-sdk v0.2.2/go.mod h1:1klR7WqfFijmd4msUvhRHoGstteUfBsRuRdX4CIJ/so=
111111
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
112112
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
113113
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

‎backend/pkg/rag/ct.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,16 @@ func (c *CTRag) UpdateDocumentMetadata(ctx context.Context, datasetID string, do
173173
return nil
174174
}
175175

176+
func (c *CTRag) ReindexDocument(ctx context.Context, datasetID string, docID string) error {
177+
_, err := c.client.Documents.Reindex(ctx, datasetID, docID)
178+
if err != nil {
179+
return err
180+
}
181+
182+
c.logger.WithContext(ctx).With("dataset_id", datasetID).With("doc_id", docID).Debug("reindex rag document")
183+
return nil
184+
}
185+
176186
func (c *CTRag) GetModelList(ctx context.Context) ([]*model.LLM, error) {
177187
list, err := c.client.Models.List(ctx, &raglite.ListModelsRequest{})
178188
if err != nil {

‎backend/pkg/rag/rag.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type Service interface {
4848
DeleteDataset(ctx context.Context, datasetID string) error
4949
DatasetProcessFinish(ctx context.Context, datasetID string) (bool, error)
5050
UpdateDocumentMetadata(ctx context.Context, datasetID string, docID string, metadata Metadata) error
51+
ReindexDocument(ctx context.Context, datasetID string, docID string) error
5152

5253
GetModelList(ctx context.Context) ([]*model.LLM, error)
5354
AddModel(ctx context.Context, model *model.LLM) (string, error)

0 commit comments

Comments
 (0)