Support file level ingestion and file listing search

frontend/app/api/mutations/useS3ConfigureMutation.ts

@@ -1,33 +1,33 @@
 import { useMutation, useQueryClient } from "@tanstack/react-query";
 
 export interface S3ConfigurePayload {
-    access_key?: string;
-    secret_key?: string;
-    endpoint_url?: string;
-    region?: string;
-    bucket_names?: string[];
-    connection_id?: string;
+  access_key?: string;
+  secret_key?: string;
+  endpoint_url?: string;
+  region?: string;
+  bucket_names?: string[];
+  connection_id?: string;
 }
 
 async function configureS3(payload: S3ConfigurePayload) {
-    const res = await fetch("/api/connectors/aws_s3/configure", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify(payload),
-    });
-    const data = await res.json();
-    if (!res.ok) throw new Error(data.error || "Failed to configure S3");
-    return data as { connection_id: string; status: string };
+  const res = await fetch("/api/connectors/aws_s3/configure", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(payload),
+  });
+  const data = await res.json();
+  if (!res.ok) throw new Error(data.error || "Failed to configure S3");
+  return data as { connection_id: string; status: string };
 }
 
 export function useS3ConfigureMutation() {
-    const queryClient = useQueryClient();
+  const queryClient = useQueryClient();
 
-    return useMutation({
-        mutationFn: configureS3,
-        onSuccess: () => {
-            queryClient.invalidateQueries({ queryKey: ["connectors"] });
-            queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
-        },
-    });
+  return useMutation({
+    mutationFn: configureS3,
+    onSuccess: () => {
+      queryClient.invalidateQueries({ queryKey: ["connectors"] });
+      queryClient.invalidateQueries({ queryKey: ["s3-defaults"] });
+    },
+  });
 }

frontend/app/api/queries/useBrowseConnectionFiles.ts

@@ -0,0 +1,75 @@
+import {
+  type UseQueryOptions,
+  useQuery,
+  useQueryClient,
+} from "@tanstack/react-query";
+
+export interface RemoteFile {
+  id: string;
+  name: string;
+  bucket: string;
+  key: string;
+  size: number;
+  modified_time: string;
+  is_ingested: boolean;
+}
+
+export interface BrowseConnectionFilesParams {
+  connectorType: string;
+  connectionId: string;
+  bucket?: string;
+  search?: string;
+  pageToken?: string;
+  maxFiles?: number;
+}
+
+export interface BrowseConnectionFilesResponse {
+  files: RemoteFile[];
+  next_page_token: string | null;
+  total_remote: number;
+  total_ingested: number;
+}
+
+export const useBrowseConnectionFiles = (
+  params: BrowseConnectionFilesParams,
+  options?: Omit<
+    UseQueryOptions<BrowseConnectionFilesResponse>,
+    "queryKey" | "queryFn"
+  >,
+) => {
+  const queryClient = useQueryClient();
+
+  async function fetchFiles(): Promise<BrowseConnectionFilesResponse> {
+    const searchParams = new URLSearchParams();
+
+    if (params.bucket) searchParams.set("bucket", params.bucket);
+    if (params.search) searchParams.set("search", params.search);
+    if (params.pageToken) searchParams.set("page_token", params.pageToken);
+    if (params.maxFiles) searchParams.set("max_files", String(params.maxFiles));
+
+    const url = `/api/connectors/${params.connectorType}/${params.connectionId}/browse?${searchParams.toString()}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      const errorData = await response
+        .json()
+        .catch(() => ({ error: "Unknown error" }));
+      throw new Error(
+        errorData.error || `Failed to browse files: ${response.status}`,
+      );
+    }
+
+    return response.json();
+  }
+
+  return useQuery(
+    {
+      queryKey: ["browseConnectionFiles", params],
+      queryFn: fetchFiles,
+      retry: false,
+      enabled: Boolean(params.connectorType && params.connectionId),
+      ...options,
+    },
+    queryClient,
+  );
+};

frontend/app/api/queries/useListFiles.ts

@@ -0,0 +1,97 @@
+import {
+  type UseQueryOptions,
+  useQuery,
+  useQueryClient,
+} from "@tanstack/react-query";
+import type { File } from "./useGetSearchQuery";
+
+export interface ListFilesParams {
+  page?: number;
+  pageSize?: number;
+  sortBy?: string;
+  sortOrder?: "asc" | "desc";
+  connectorType?: string;
+  mimetype?: string;
+  owner?: string;
+  search?: string;
+}
+
+export interface ListFilesResponse {
+  files: File[];
+  total: number;
+  page: number;
+  page_size: number;
+}
+
+export const useListFiles = (
+  params: ListFilesParams = {},
+  options?: Omit<UseQueryOptions<ListFilesResponse>, "queryKey" | "queryFn">,
+) => {
+  const queryClient = useQueryClient();
+
+  async function fetchFiles(): Promise<ListFilesResponse> {
+    const searchParams = new URLSearchParams();
+
+    if (params.page) searchParams.set("page", String(params.page));
+    if (params.pageSize) searchParams.set("page_size", String(params.pageSize));
+    if (params.sortBy) searchParams.set("sort_by", params.sortBy);
+    if (params.sortOrder) searchParams.set("sort_order", params.sortOrder);
+    if (params.connectorType)
+      searchParams.set("connector_type", params.connectorType);
+    if (params.mimetype) searchParams.set("mimetype", params.mimetype);
+    if (params.owner) searchParams.set("owner", params.owner);
+    if (params.search) searchParams.set("search", params.search);
+
+    const url = `/api/files?${searchParams.toString()}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      const errorData = await response
+        .json()
+        .catch(() => ({ error: "Unknown error" }));
+      throw new Error(
+        errorData.error || `Failed to list files: ${response.status}`,
+      );
+    }
+
+    const data = await response.json();
+
+    // Map server response to File interface
+    const files: File[] = (data.files || []).map(
+      (f: Record<string, unknown>) => ({
+        filename: (f.filename as string) || "",
+        mimetype: (f.mimetype as string) || "",
+        chunkCount: (f.chunk_count as number) || 0,
+        source_url: (f.source_url as string) || "",
+        owner: (f.owner as string) || "",
+        owner_name: (f.owner_name as string) || "",
+        owner_email: (f.owner_email as string) || "",
+        size: (f.file_size as number) || 0,
+        connector_type: (f.connector_type as string) || "local",
+        embedding_model: f.embedding_model as string | undefined,
+        embedding_dimensions: f.embedding_dimensions as number | undefined,
+        allowed_users: (f.allowed_users as string[]) || [],
+        allowed_groups: (f.allowed_groups as string[]) || [],
+        status: "active" as const,
+      }),
+    );
+
+    return {
+      files,
+      total: data.total || 0,
+      page: data.page || 1,
+      page_size: data.page_size || 25,
+    };
+  }
+
+  return useQuery(
+    {
+      queryKey: ["listFiles", params],
+      placeholderData: (prev: ListFilesResponse | undefined) => prev,
+      queryFn: fetchFiles,
+      retry: false,
+      ...options,
+    },
+    queryClient,
+  );
+};

frontend/app/api/queries/useS3DefaultsQuery.ts

@@ -1,25 +1,25 @@
 import { useQuery } from "@tanstack/react-query";
 
 export interface S3Defaults {
-    access_key_set: boolean;
-    secret_key_set: boolean;
-    endpoint: string;
-    region: string;
-    bucket_names: string[];
-    connection_id: string | null;
+  access_key_set: boolean;
+  secret_key_set: boolean;
+  endpoint: string;
+  region: string;
+  bucket_names: string[];
+  connection_id: string | null;
 }
 
 async function fetchS3Defaults(): Promise<S3Defaults> {
-    const res = await fetch("/api/connectors/aws_s3/defaults");
-    if (!res.ok) throw new Error("Failed to fetch S3 defaults");
-    return res.json();
+  const res = await fetch("/api/connectors/aws_s3/defaults");
+  if (!res.ok) throw new Error("Failed to fetch S3 defaults");
+  return res.json();
 }
 
 export function useS3DefaultsQuery(options?: { enabled?: boolean }) {
-    return useQuery<S3Defaults>({
-        queryKey: ["s3-defaults"],
-        queryFn: fetchS3Defaults,
-        enabled: options?.enabled ?? true,
-        staleTime: 0,
-    });
+  return useQuery<S3Defaults>({
+    queryKey: ["s3-defaults"],
+    queryFn: fetchS3Defaults,
+    enabled: options?.enabled ?? true,
+    staleTime: 0,
+  });
 }