Improve accuracy

.gitignore

@@ -1,9 +1,14 @@
 .env
 venv/*
 *.csv
+*.xlsx
 **/__pycache__
 **egg-info
 *.DS_Store
 *ipynb_checkpoints
 *.vscode
-.github/instructions/
+.github/instructions/
+.idea/
+Davis_atamis_test.xlsx
+nhs_data_evaluation.py
+tenders_experiment.ipynb

async_investigate_model_accuracy.py

@@ -0,0 +1,94 @@
+import pandas as pd
+import asyncio
+import ast
+from pathlib import Path
+from src.core.classification_v2_mix_v5 import keywords_and_llm
+
+# --- Configuration & Data Loading ---
+file_path = "AI Catrgorisation Testing Notes2.xlsx"
+data = pd.read_excel(file_path)
+
+# Cleaning logic based on your specific row indices
+drop_1 = data.iloc[0:8].index
+drop_2 = data.iloc[170:181].index
+all_to_drop = drop_1.union(drop_2)
+new_data = data.drop(all_to_drop)
+print("length:", len(new_data))
+# Filter for labeled data and take a sample for testing
+data_to_analyse = new_data[new_data["Correct Match "].notna()].iloc[0:781].copy()
+
+
+async def run_classification_test(df):
+    """
+    Async loop to process each row through the Hybrid Classifier.
+    """
+    output_labels = []
+    output_reasons = []
+
+    print(f"Starting analysis on {len(df)} rows...")
+
+    for index, row in df.iterrows():
+        print(f"--- Processing Row: {index} ---")
+
+        # 1. Safely parse the ContractDescription dictionary string
+        try:
+            description_raw = row["ContractDescription"]
+            # Handle cases where it might already be a dict or needs parsing
+            if isinstance(description_raw, str):
+                description_dict = ast.literal_eval(description_raw)
+            else:
+                description_dict = description_raw
+
+            clean_desc = description_dict.get("description", "")
+        except (ValueError, SyntaxError, AttributeError) as e:
+            print(f"Warning: Could not parse description at index {index}: {e}")
+            clean_desc = str(row["ContractDescription"])
+
+        # 2. Build the combined text for the classifier
+        # We combine title and description for maximum keyword context
+        contract_text = f"{row['contract_title']} : {clean_desc}"
+
+        # 3. Call the Hybrid Classifier (AWAIT required for the LLM fallback)
+        # result = category label, reason = "Heuristic Match" or "LLM Categorization"
+        result, reason = await keywords_and_llm(contract_text)
+
+        output_labels.append(result)
+        output_reasons.append(reason)
+
+        print(f"AI Prediction: {result}")
+        print(f"Actual Label:  {row['Correct Match ']}")
+        print(f"Method:        {reason}\n")
+
+    # 4. Update the DataFrame and save
+    df["AI_CategoryMatchV3"] = output_labels
+    df["Classification_Method"] = output_reasons
+
+    # Accuracy Calculations
+    accuracy_series = df["AI_CategoryMatchV3"] == df["Correct Match "]
+    correct_count = accuracy_series.sum()
+    total_count = len(df)
+    accuracy_pct = (correct_count / total_count) * 100
+    old_model_correct_df = df[df["AI_CategoryMatch"] == df["Correct Match "]]
+    print(f"--- Test Results ---")
+    print(f"Total Analyzed: {total_count}")
+    print(f"Correct Matches: {correct_count}")
+    print(f"Accuracy: {accuracy_pct:.2f}%")
+    print(f"Old model total correct matches: {len(old_model_correct_df)}")
+    print(f"Old model total correct accuracy%: {len(old_model_correct_df)/total_count*100}%")
+
+    # Save to Excel
+    df = df[[
+        "contract_title",
+        "ContractDescription",
+        "Correct Match ",
+        "AI_CategoryMatch",
+        "AI_CategoryMatchV3",
+    ]]
+    output_file = "new_AI_results_clean_data.xlsx"
+    df.to_excel(output_file, index=False)
+    print(f"Results saved to {output_file}")
+
+
+if __name__ == "__main__":
+    # This entry point starts the asyncio event loop
+    asyncio.run(run_classification_test(data_to_analyse))

investigate_model_accuracy.py

@@ -0,0 +1,46 @@
+import  pandas as pd
+
+from src.core.classification_v3_mix_v5 import keywords_and_llm
+from src.core.classification_v3 import contract_mapper_v3
+import ast
+
+data = pd.read_excel("AI Catrgorisation Testing Notes2.xlsx")
+drop_1 = data.iloc[0:8].index# remove rows where it does not matter what label the model gives
+drop_2 = data.iloc[170:181].index # remove rows where it does not matter what label the model gives
+
+all_to_drop = drop_1.union(drop_2)
+new_data = data.drop(all_to_drop)
+
+
+data_to_analyse = new_data[new_data["Correct Match "].notna()].iloc[0:669]
+
+
+print(len(data_to_analyse))
+# todo apply RAG to the data see if can label
+# data_to_analyse["AI_CategoryMatchV3"] =
+output = []
+for index, row in data_to_analyse.iterrows():
+    print("row:",index)
+
+    description = ast.literal_eval(row["ContractDescription"])
+    print("contract description:", description["description"])
+    contract = f"""
+    {row["contract_title"]} : {description["description"]}
+
+    """
+
+    result, score = keywords_and_llm(contract)
+    output.append(result)
+    print("AI:",result ,"Actual:", row["Correct Match "])
+
+
+data_to_analyse["AI_CategoryMatchV3"] =output
+data_to_analyse.to_excel("new_AI_results.xlsx", index=False)
+accuracy = data_to_analyse["AI_CategoryMatchV3"] == data_to_analyse["Correct Match "]
+
+correct_count = accuracy.sum()
+total_count = len(accuracy)
+old_model_correct_df = data_to_analyse[data_to_analyse["AI_CategoryMatch"] == data_to_analyse["Correct Match "]]
+print(f"Total Analyzed: {total_count}")
+print(f"Correct Matches: {correct_count}, Old model total correct matches: {len(old_model_correct_df)/total_count*100}%")
+print(f"Correct Matches %: {(correct_count/total_count)*100}, Old model total correct matches%: {len(old_model_correct_df)/total_count*100}%")

prompts/contractmap_prompt_with_descriptions.md

@@ -24,8 +24,6 @@ You are a system that categorises contracts based on their descriptions. The val
 
 * **Facilities Management**: Buying Better Food and Drink, Courier and Specialist Movements, Estate Management Services, Facilities Management and Workplace Services, Facilities Management and Workplace Services DPS, Furniture and Associated Services, Healthcare Soft FM, Housing Maintenance and Repair, Logistics, Warehousing and Supply Chain Solutions, Office Solutions, Postal Services & Solutions, Security - Physical, Technical and Support Services, Storage, Distribution, Kitting and Associated Services
 
-* **Low Value**
-
 * **Cloud and Hosting**: Cloud Compute, Crown Hosting ll, G-Cloud4; G-Cloud4 Lot
 
 * **Digital and Technology Services**: Cyber Security Services; Digital & Legacy Spplication Services, Digital Capability for Health, Digital Capability for Health, Digital Outcomes; Digital Specialists and Programmes, Digital and IT Professional Services, Quality Assurance and Testing for IT Systems, Software Design and Implementation Services, Spark DPS, Technology Services, Transport Technology & Associated Services, Vehicle Charging Infrastructure Solutions

prompts/keyword_system_prompt_v2.md

@@ -0,0 +1,36 @@
+# Role
+You are a Senior Procurement Taxonomy Architect. Your expertise is in "Feature Extraction"—identifying unique identifiers that distinguish one contract category from another in a complex regulatory environment.
+
+# Task
+Analyze the provided collection of framework titles, summaries, and service lots for the category: {{category_name}}. 
+
+Your goal is to extract a set of "Semantic Anchors" (Keywords and Codes) to be used in a high-precision heuristic classifier.
+
+# Extraction Strategy
+
+## 1. Primary Anchors (High Precision / 3 points)
+- **RM Numbers (CRITICAL)**: Extract every "RM" followed by 4 digits (e.g., **RM6116**, **RM3808**). These are the strongest identifiers and MUST be assigned to the Primary list.
+- **Technical Nouns**: Industry-specific equipment or regulatory terms exclusive to this category.
+- **Examples**: 
+    - Energy: **RM6305**, **kerosene**, **biomass**, **literage**.
+    - Financial Services: **RM6186**, **acquiring**, **pisp**, **aisp**, **mastercard**.
+    - Facilities Management: **RM6232**, **asbestos**, **pest**, **locksmith**.
+
+## 2. Secondary Anchors (Supporting Context / 1 point)
+- **Definition**: Words that are common in this domain but might overlap with others.
+- **Goal**: These provide "evidence" when combined with other words.
+- **Examples**: **supply**, **metering**, **validation**, **transaction**, **maintenance**, **repair**.
+
+# Negative Constraints (The "Noise" Filter)
+DO NOT extract any of the following generic procurement "Stop Words":
+- **Administrative**: provision, contract, framework, agreement, services, supply, public, sector, national, regional, local.
+- **Vague Descriptors**: strategic, project, solution, management, support, standard, professional, bespoke.
+- **Verbs/Adjectives**: providing, managing, delivering, complex, general.
+
+# Formatting Instructions
+- Return the results in a FLAT LIST of strings (Pydantic `List[str]`).
+- Focus on NOUNS and RM CODES.
+- Maximum 20 Primary Anchors and 20 Secondary Anchors per category.
+
+# Input Data for Analysis
+{{combined_text}}

prompts/new_system_prompt.md

@@ -26,8 +26,6 @@ If the contract does not fit any category, respond with **Outside New Taxonomy**
 
 * **Facilities Management**: Building maintenance, cleaning, security, logistics, furniture, healthcare soft FM.
 
-* **Low Value**: Low monetary value contracts not classified elsewhere.
-
 * **Cloud and Hosting**: Cloud computing, hosting, G-Cloud framework services.
 
 * **Digital and Technology Services**: Digital transformation, cybersecurity, IT services, software testing and development.

prompts/summariser_system_prompt.md

@@ -0,0 +1,26 @@
+# Role
+You are a Legal and Procurement Data Analyst specializing in contract taxonomy. Your task is to transform complex, jargon-heavy framework descriptions into high-clarity "Category Rule Cards."
+
+# Task
+Analyze the provided "Label Context" and produce a three-part summary:
+1. **The Core Essence**: A 1-2 sentence high-level summary of the category.
+2. **Inclusions (The "Yes" List)**: A bulleted list of specific products, services, or frameworks that belong in this category.
+3. **Exclusions (The "No" List)**: A bulleted list of specific items that are explicitly moved to other categories or are out of scope.
+
+# Formatting Rules
+- Use **bolding** for specific product names or technical terms (e.g., **Apprenticeship Levy**, **SaaS**).
+- Keep descriptions concise; remove administrative "noise" like dates, PCR regulations, or video links unless they define the scope.
+- If a specific "RM" number (Framework Reference) is mentioned, include it in the inclusions.
+- If no exclusions are mentioned, write "None explicitly stated."
+
+# Output Template
+### Category Summary
+[1-2 sentences here]
+
+### Key Inclusions
+* item 1
+* item 2
+
+### Key Exclusions
+* item 1
+* item 2

prompts/system_prompt.md

@@ -26,8 +26,6 @@ If the contract does not fit any category, respond with **Outside New Taxonomy**
 
 * **Facilities Management**: Building maintenance, cleaning, security, logistics, furniture, healthcare soft FM.
 
-* **Low Value**: Low monetary value contracts not classified elsewhere.
-
 * **Cloud and Hosting**: Cloud computing, hosting, G-Cloud framework services.
 
 * **Digital and Technology Services**: Digital transformation, cybersecurity, IT services, software testing and development.

prompts/system_prompt_v2.md

@@ -29,8 +29,6 @@ You are an expert contract categorization system, designed for high-accuracy, si
 
 * **Facilities Management**: Building maintenance (Hard FM: HVAC, electrical, plumbing), cleaning, security, logistics, furniture, maintenance, repair, and ongoing support of specialized operational equipment (e.g., medical devices, lab equipment, manufacturing machinery), healthcare soft FM.
 
-* **Low Value**: (LAST RESORT ONLY) Low monetary value contracts not classified elsewhere. Do NOT use if a specific category (e.g., HR, Energy) clearly applies.
-
 * **Cloud and Hosting**: Cloud computing, hosting, G-Cloud framework services (XaaS).
 
 * **Digital and Technology Services**: Digital transformation, cybersecurity, IT services, software testing and development (High-level strategy and bespoke software services).
@@ -49,9 +47,7 @@ You are an expert contract categorization system, designed for high-accuracy, si
 
 2. **Network Services Priority (Connectivity):** If the work involves the **installation, maintenance, or management of core data/telecommunications network infrastructure** (e.g., Cat 5e/6, fibre optic cabling, W-LAN/LAN management), the connectivity function outweighs physical installation/logging. Classify as **Network Services**.
 
-3. **Low Value**: Apply the rules defined in the category definitions.
-
-4. **Implied Support:** Use context clues (e.g., "managed services," "maintenance agreement," "as-a-service") to determine whether ongoing support is implied.
+3. **Implied Support:** Use context clues (e.g., "managed services," "maintenance agreement," "as-a-service") to determine whether ongoing support is implied.
 
 ---
 

prompts/system_prompt_v3.md

@@ -0,0 +1,23 @@
+ACT AS A DATA CLASSIFIER.
+
+CONTRACT DESCRIPTION:
+{{contract_description}}
+
+
+RULES TO FOLLOW:
+{{relevant_rules}}
+
+# EXTRACTION RULE
+In the "RULES TO FOLLOW" section, each rule begins with a header formatted as:
+"--- CATEGORY: [CATEGORY_NAME] ---"
+
+# TASK
+1. Analyze the CONTRACT DESCRIPTION against the provided RULES.
+2. Identify the single best matching CATEGORY from the provided rules.
+3. Extract ONLY the [CATEGORY_NAME] from that rule's header.
+4. If the CONTRACT DESCRIPTION does not fit any of the provided CATEGORIES, or if it matches an 'Exclusion' for all provided categories, return "Outside New Taxonomy".
+
+# CRITICAL CONSTRAINTS
+- Return ONLY the clean [CATEGORY_NAME] (e.g., "Software", "Construction").
+- DO NOT include the Framework Title, RM numbers, or any text appearing after the header (e.g., do NOT return "Software - Back Office Software 2 (RM6285)").
+- Your response must be a single phrase/category name with no preamble or punctuation.

prompts/system_prompts.py

@@ -23,8 +23,6 @@ def system_prompt():
 \n
 * 'Facilities Management': Building maintenance, cleaning, security, logistics, furniture, healthcare soft FM.
 \n
-* 'Low Value': Low monetary value contracts not classified elsewhere.
-\n
 * 'Cloud and Hosting': Cloud computing, hosting, G-Cloud framework services.
 \n
 * 'Digital and Technology Services': Digital transformation, cybersecurity, IT services, software testing and development.
@@ -65,8 +63,7 @@ def system_prompt_v2():
 * 'Travel, Accommodation and Venues': Travel arrangements, accommodation, and venue services.
 * 'Construction': Building works, emergency repairs, architectural and engineering services related to building and infrastructure projects, materials supply (Use if the primary scope is the *building* itself, not the equipment inside).
 * 'Energy': Electricity, gas, fuel supply, power purchase agreements, water, wastewater services.
-* **'Facilities Management': Building maintenance (Hard FM: HVAC, electrical, plumbing), cleaning, security, logistics, furniture, maintenance, repair, and ongoing support of specialized operational equipment (e.g., medical devices, lab equipment, manufacturing machinery), healthcare soft FM.**
-* **'Low Value': (LAST RESORT ONLY)** Low monetary value contracts not classified elsewhere. Do NOT use if a specific category (e.g., HR, Energy) clearly applies.
+* **'Facilities Management': Building maintenance (Hard FM: HVAC, electrical, plumbing), cleaning, security, logistics, furniture, maintenance, repair, and ongoing support of specialized operational equipment (e.g., medical devices, lab equipment, manufacturing machinery), healthcare soft FM.
 * 'Cloud and Hosting': Cloud computing, hosting, G-Cloud framework services (XaaS).
 * 'Digital and Technology Services': Digital transformation, cybersecurity, IT services, software testing and development (High-level strategy and bespoke software services).
 * **'Network Services': Installation, management, and support of passive (cabling) and active (switches, routers) network infrastructure,** audiovisual consultancy, network connectivity, mobile/data services.

requirements.in

@@ -0,0 +1,31 @@
+# Base Tools (Add these!)
+pip
+pip-tools
+
+# Core Framework
+langchain
+langchain-core
+langchain-community
+langchain-openai
+pydantic-ai-slim[openai,google,groq]==1.59.0
+
+# Environment & Data
+python-dotenv
+pandas
+openpyxl
+
+# API & Server
+fastapi
+uvicorn
+git+https://github.com/DavisAgyemang/ccss_web_api_tool.git
+
+# Azure Integration
+azureml-mlflow
+azure-ai-ml
+azure-identity
+azure-search-documents
+
+# Development & Testing
+pytest
+ruff
+pre-commit