add two examples and corresponding integration tests

Author: Magdalen Manohar

diskann-benchmark/example/graph-index-dynamic-yfcc.json

@@ -0,0 +1,56 @@
+{
+  "search_directories": [
+    "test_data/yfcc"
+  ],
+  "jobs": [
+    {
+      "type": "graph-index-dynamic-run",
+      "content": {
+        "build": {
+          "data_type": "float32",
+          "data": "yfcc_10k.fbin",
+          "distance": "squared_l2",
+          "max_degree": 32,
+          "l_build": 50,
+          "alpha": 1.2,
+          "backedge_ratio": 1.0,
+          "num_threads": 4,
+          "start_point_strategy": "medoid"
+        },
+        "search_phase": {
+            "search-type": "topk",
+            "queries": "yfcc_query_100.fbin",
+            "groundtruth": "groundtruth.bin",
+            "reps": 1,
+            "num_threads": [
+              2
+            ],
+            "runs": [
+              {
+                "search_n": 10,
+                "search_l": [
+                  20,
+                  40
+                ],
+                "recall_k": 10
+              }
+            ]
+        },
+        "runbook_params": {
+            "runbook_path": "yfcc_runbook.yaml",
+            "dataset_name": "yfcc-10K",
+            "gt_directory": "yfcc_runbook_gt",
+            "ip_delete_method": {
+              "method": "visited_and_top_k",
+              "params": {
+                "k_value": 20,
+                "l_value": 50
+              }
+            },
+            "ip_delete_num_to_replace": 3,
+            "consolidate_threshold": 0.2
+          }
+      }
+    }
+  ]
+}

diskann-benchmark/example/graph-index-inline-filter-yfcc.json

@@ -0,0 +1,52 @@
+{
+  "search_directories": [
+    "test_data/yfcc"
+  ],
+  "jobs": [
+    {
+      "type": "graph-index-build",
+      "content": {
+        "source": {
+          "index-source": "Build",
+          "data_type": "float32",
+          "data": "yfcc_10k.fbin",
+          "distance": "squared_l2",
+          "max_degree": 32,
+          "l_build": 50,
+          "alpha": 1.2,
+          "backedge_ratio": 1.0,
+          "num_threads": 1,
+          "start_point_strategy": "medoid",
+          "num_insert_attempts": 1,
+          "saturate_inserts": false
+        },
+        "search_phase": {
+          "search-type": "topk-inline-filter",
+          "queries": "yfcc_query_100.fbin",
+          "groundtruth": "groundtruth_filtered.rangeres",
+          "query_predicates": "yfcc_query_filters.json",
+          "data_labels": "yfcc_metadata.json",
+          "reps": 5,
+          "num_threads": [
+            1
+          ],
+          "runs": [
+            {
+              "search_n": 20,
+              "search_l": [
+                20,
+                30,
+                40
+              ],
+              "recall_k": 10
+            }
+          ],
+          "adaptive_l": {
+            "sample_count": 10,
+            "scale_factor": 16.0
+          }
+        }
+      }
+    }
+  ]
+}

diskann-benchmark/src/main.rs

@@ -318,6 +318,12 @@ mod tests {
         run_integration_test(raw);
     }
 
+    #[test]
+    fn graph_index_dynamic_yfcc_integration() {
+        let raw = value_from_file(&example_directory().join("graph-index-dynamic-yfcc.json"));
+        run_integration_test(raw);
+    }
+
     ////////////////////////////
     //     BF-Tree Index      //
     ////////////////////////////
@@ -587,6 +593,14 @@ mod tests {
         run_integration_test(raw);
     }
 
+    #[test]
+    fn graph_index_inline_filter_yfcc_integration() {
+        // First, parse and modify the input file to establish paths relative to the
+        // directory building the dispatcher.
+        let raw = value_from_file(&example_directory().join("graph-index-inline-filter-yfcc.json"));
+        run_integration_test(raw);
+    }
+
     #[test]
     fn graph_index_filter_integration_with_gt_compute() {
         let storage_provider = FileStorageProvider;