Skip to content

Fix for data source name and size blank #564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shreyas-damle wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix for data source name and size blank #564

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6524d38
- Fixed data source name missing on Dashboard => Safe Loader => Docum…
shreyas-damle Sep 20, 2024
ac1e3a2
Ruff fix.
shreyas-damle Sep 20, 2024
7431b27
Riff fix.
shreyas-damle Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pebblo/app/models/db_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ class AiDataSource(BaseModel):
loadId: str
metadata: Metadata
sourcePath: str
sourceSize: int
sourceType: str
loader: str

Expand Down
1 change: 1 addition & 0 deletions pebblo/app/service/loader/loader_doc_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ def _get_or_create_data_source(self):
"modifiedAt": get_current_time(),
},
"sourcePath": loader_details.get("source_path"),
"sourceSize": loader_details.get("source_aggregate_size", 0),
"sourceType": loader_details.get("source_type"),
"loader": loader_details.get("loader"),
}
Expand Down
17 changes: 17 additions & 0 deletions pebblo/app/service/local_ui/loader_apps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ def _update_loader_datasource(
"name": ds_data["loader"],
"sourcePath": ds_data["sourcePath"],
"sourceType": ds_data["sourceType"],
"sourceSize": ds_data.get("sourceSize", "-"),
"findingsEntities": entity_count,
"findingsTopics": topic_count,
"totalSnippetCount": total_snippet_count,
Expand All @@ -246,6 +247,19 @@ def _update_loader_datasource(
self.loader_details.get("loader_data_source_list", [])
)

def _get_data_source_name(self, document_detail: dict):
"""
Get data source name for given data source id from db.
"""
data_source_id = document_detail.get("dataSourceId")
source_name = "-"
if data_source_id:
_, datasource = self.db.query(AiDataSourceTable, {"id": data_source_id})
if datasource:
source_name = datasource[0].data.get("loader", "-")

return source_name

def _get_documents_with_findings(self, app_data: AiDataLoaderTable) -> None:
"""
Fetch required data for DocumentWithFindings
Expand All @@ -268,9 +282,12 @@ def _get_documents_with_findings(self, app_data: AiDataLoaderTable) -> None:
for topic, topic_data in document_detail.get("topics", {}).items():
topic_count += topic_data.get("count", 0)

source_name = self._get_data_source_name(document_detail)

if document_detail["sourcePath"] in loader_document_with_findings:
document_obj = {
"appName": document_detail["appName"],
"sourceName": source_name,
"owner": document_detail.get("owner", "-"),
"sourceSize": document_detail.get("sourceSize", 0),
"sourceFilePath": document_detail["sourcePath"],
Expand Down