Update politeness v2

src/team_comm_tools/features/politeness_v2_helper.py

@@ -238,8 +238,6 @@ def wh_is_real_question(tok, sent, auxiliaries, ends_with_question_mark=False):
     if len(sent_tokens) >= 1:
         first_tok = sent_tokens[0]
         if first_tok.text.lower() in auxiliaries and first_tok.text.lower() not in {'what', 'who', 'where', 'when', 'why', 'how', 'which'}:
-            # Sentence starts with auxiliary - this is a Yes/No question
-            # Any WH-words are being used as content, not interrogatives
             return False
 
     # For WH-determiners (both with and without ?), use special logic
@@ -280,10 +278,58 @@ def wh_is_real_question(tok, sent, auxiliaries, ends_with_question_mark=False):
         return False
 
     # For other WH-words (not determiners)
-    # First check for complement clauses (ccomp, xcomp) - these are embedded questions
-    for anc in tok.ancestors:
-        if anc.dep_ in {"ccomp", "xcomp"}:
+
+    # Special handling for WH-words that are subjects (nsubj, nsubjpass)
+    if tok.dep_ in {"nsubj", "nsubjpass"}:
+        # First check: is this part of a relative clause?
+        has_relcl_ancestor = False
+        for anc in tok.ancestors:
+            if anc.dep_ == "relcl":
+                has_relcl_ancestor = True
+                break
+
+        if has_relcl_ancestor:
+            # Check if there's a noun before the WH-word (ignoring punctuation)
+            # This is the typical relative clause pattern: "the book, which..."
+            has_noun_before = False
+            for t in sent:
+                if t.i >= tok.i:
+                    break
+                if t.pos_ in {"NOUN", "PROPN"}:
+                    has_noun_before = True
+
+            # If there's a noun before WH and it has relcl ancestor, it's a real relative clause
+            if has_noun_before:
+                return False
+
+        # Check if there's actually a complement-taking verb before the WH-word
+        complement_taking_verbs = {
+            'tell', 'ask', 'know', 'wonder', 'understand', 'explain', 
+            'show', 'see', 'remember', 'forget', 'realize', 'figure',
+            'decide', 'consider', 'discover', 'find', 'learn', 'teach'
+        }
+
+        has_complement_verb_before = False
+        for t in sent:
+            if t.i >= tok.i:
+                break
+            if t.pos_ == "VERB" and t.lemma_ in complement_taking_verbs:
+                has_complement_verb_before = True
+                break
+
+        # If no complement-taking verb before WH-word, it's a main question
+        if not has_complement_verb_before:
+            # Check for an auxiliary after the WH-word
+            for t in sent:
+                if t.i > tok.i and t.text.lower() in auxiliaries:
+                    return True
             return False
+        # If there IS a complement verb, fall through to normal checks
+
+        # First check for complement clauses (ccomp, xcomp) - these are embedded questions
+        for anc in tok.ancestors:
+            if anc.dep_ in {"ccomp", "xcomp"}:
+                return False
 
     # Check if WH-word is attached to a verb that takes interrogative complements
     complement_taking_verbs = {
@@ -333,7 +379,8 @@ def wh_is_real_question(tok, sent, auxiliaries, ends_with_question_mark=False):
     if ends_with_question_mark:
         return True
 
-    # For non-? sentences with non-determiner WH-words
+    # For non-? sentences with non-determiner WH-words that are NOT nsubj
+    # (nsubj was already handled above)
     if is_in_subordinate_clause(tok, sent):
         return False
 
@@ -370,7 +417,7 @@ def Question(doc):
         sent_tokens = list(sent)
         if not sent_tokens:
             continue
-            
+
         # Method 1: Sentences ending with '?'
         if sent_text.endswith('?'):
             wh = False
@@ -397,7 +444,7 @@ def Question(doc):
                     counted_sentences.add(sent.start)
                     found_question = True
                     break
-        
+
         if found_question:
             continue
 

tests/data/cleaned_data/test_chat_level.csv

@@ -88,14 +88,38 @@ I respond to that too",num_block_quote_responses,2
 1,A,hello,Hello_receptiveness_yeomans,1
 1,B,So how should we answer this,Token_count_receptiveness_yeomans,6
 1,A,We can start here. What is the question?,YesNo_Questions_receptiveness_yeomans,0
-1,B,I am not sure. Where is the rest of our team?,WH_Questions_receptiveness_yeomans,1
 1,B,"Please help me figure this out, I really want to do well on this please",Please_receptiveness_yeomans,2
+1,B,I am not sure. Where is the rest of our team?,WH_Questions_receptiveness_yeomans,1
+1,B,I am not sure. Where is the rest of our team?,First_Person_Single_receptiveness_yeomans,1
+1,B,"Well please help me figure this out, I really want to do well on this please okay",factuality_politeness_convokit,1
+1B,A,is where different from why?,YesNo_Questions_receptiveness_yeomans,1
+1B,B,is where different from why?,WH_Questions_receptiveness_yeomans,0
+1B,A,could where be different from why?,YesNo_Questions_receptiveness_yeomans,1
+1B,B,could where be different from why?,WH_Questions_receptiveness_yeomans,0
+1B,A,ok so I don't really understand which one it is,YesNo_Questions_receptiveness_yeomans,0
+1B,B,ok so I don't really understand which one it is,WH_Questions_receptiveness_yeomans,0
+1B,A,"the book, which is on the table, is red",YesNo_Questions_receptiveness_yeomans,0
+1B,B,"the book, which is on the table, is red",WH_Questions_receptiveness_yeomans,0
+1B,A,"The love which we had long sought for has finally arrived.",YesNo_Questions_receptiveness_yeomans,0
+1B,B,"The love which we had long sought for has finally arrived.",WH_Questions_receptiveness_yeomans,0
+1B,A,alrighty so which was your favorite,YesNo_Questions_receptiveness_yeomans,0
+1B,B,alrighty so which was your favorite,WH_Questions_receptiveness_yeomans,1
+1B,A,"after trying all the cake, which ones were your favorites",YesNo_Questions_receptiveness_yeomans,0
+1B,B,"after trying all the cake, which ones were your favorites",WH_Questions_receptiveness_yeomans,1
+1B,A,are these the books which you were looking for,YesNo_Questions_receptiveness_yeomans,1
+1B,B,are these the books which you were looking for,WH_Questions_receptiveness_yeomans,0
+1B,A,are these the books which you were looking for?,YesNo_Questions_receptiveness_yeomans,1
+1B,B,are these the books which you were looking for?,WH_Questions_receptiveness_yeomans,0
+1B,A,"the book, which is on the table, is red?",YesNo_Questions_receptiveness_yeomans,1
+1B,B,"the book, which is on the table, is red?",WH_Questions_receptiveness_yeomans,0
+1B,A,can you tell me who it is? what is it? where?,YesNo_Questions_receptiveness_yeomans,1
+1B,B,can you tell me who it is? what is it? where?,WH_Questions_receptiveness_yeomans,2
+1B,A,with whom did Napoleon fight his first war?,YesNo_Questions_receptiveness_yeomans,0
+1B,B,with whom did Napoleon fight his first war?,WH_Questions_receptiveness_yeomans,1
 2,C,Hey,Hello_receptiveness_yeomans,1
 2,C,Okay bro lets split it 50/50,Impersonal_Pronoun_receptiveness_yeomans,1
 2,D,Maybe but how about 60/40? I doubt its fair otherwise,Hedges_receptiveness_yeomans,2
 2,C,Seems fair,Hedges_receptiveness_yeomans,1
-1,B,I am not sure. Where is the rest of our team?,First_Person_Single_receptiveness_yeomans,1
-1,B,"Well please help me figure this out, I really want to do well on this please okay",factuality_politeness_convokit,1
 2,C,Seems possible,hashedge_politeness_convokit,1
 2,E,I see what youre thinking but I disagree,Acknowledgement_receptiveness_yeomans,1
 2,E,We get only one chance so we should understand how to split it,Acknowledgement_receptiveness_yeomans,2