Yes it's awesome

rapidsms_xforms/models.py

@@ -16,6 +16,7 @@
 from django.contrib.sites.managers import CurrentSiteManager
 from rapidsms.models import ExtensibleModelBase
 from eav.fields import EavSlugField
+from rapidsms_httprouter.models import Message
 
 class XForm(models.Model):
     """
@@ -32,6 +33,9 @@ class XForm(models.Model):
         (';','Semicolon (;)'),
         (':','Colon (:)'),
         ('*','Asterisk (*)'),
+        ('.','Period (.)'),
+        ('\\s','Whitespace'),
+        ('"', 'Quotation Mark'),
     )
 
     name = models.CharField(max_length=32,
@@ -51,8 +55,8 @@ class XForm(models.Model):
     keyword_prefix = models.CharField(max_length=1, choices=PREFIX_CHOICES, null=True, blank=True,
                                       help_text="The prefix required for form keywords, defaults to no prefix.")
 
-    separator = models.CharField(max_length=1, choices=SEPARATOR_CHOICES, null=True, blank=True,
-                                 help_text="The separator character for fields, field values will be split on this character.")
+    separator = models.CharField(max_length=8, null=True, blank=True,
+                                 help_text="The separator character(s) for fields, field values will be split on these characters.")
 
     owner = models.ForeignKey(User)
     created = models.DateTimeField(auto_now_add=True)
@@ -296,20 +300,25 @@ def parse_sms_submission(self, message_obj):
             submission['response'] = "Incorrect keyword.  Keyword must be '%s'" % self.keyword
             return submission
 
-        # separator mode means we don't split on spaces
-        separator = None
-
         # if the separator is some non-whitespace character, and this form has only
         # one text-type field, the entire remainder can be considered its [command]-value pair
-        if self.separator and self.separator.strip() and self.fields.count() == 1 and self.fields.all()[0].field_type == XFormField.TYPE_TEXT:
+        if self.separator and self.separator != '\\s' and self.fields.count() == 1 and self.fields.all()[0].field_type == XFormField.TYPE_TEXT:
             segments = [remainder,]
         else:
-            # figure out if we are using separators
-            if self.separator and message.find(self.separator) >= 0:
-                separator = self.separator
-
-            # so first let's split on the separator passed in
-            segments = remainder.split(separator)
+            # split up the segments based on the separators
+            segments = []
+            separators = []
+            separator = self.separator or '\\s'
+            search_str = '[%s]+' % separator
+            match = re.search(search_str, remainder)
+            while match:
+                segment = remainder[0:match.start()]
+                separator = remainder[match.start():match.end()]
+                segments.append(segment)
+                separators.append(separator)
+                remainder = remainder[match.end():]
+                match = re.search(search_str, remainder)
+            segments.append(remainder)
 
         # remove any segments that are empty
         stripped_segments = []
@@ -371,6 +380,19 @@ def parse_sms_submission(self, message_obj):
                 # pop it back on and break
                 segments.insert(0, segment)
                 break
+
+            # check for cases where a fully-alpha command and an integer value
+            # are together with no separator (a common and unambiguous typeo),
+            # e.g. 'age27'
+            if field.command.isalpha() and field.field_type == XFormField.TYPE_INT:
+                match_num = re.search('[0-9]', segment)
+                if match_num:
+                    command_segment = segment[0:match_num.start()]
+                    num_segment = segment[match_num.start():]
+                    if self.is_command(command_segment, commands) and num_segment.isdigit():
+                        segments.insert(0, num_segment)
+                        segments.insert(0, command_segment)
+                        break
 
             # ok, this looks like a valid required field value, clean it up
             try:
@@ -379,7 +401,7 @@ def parse_sms_submission(self, message_obj):
             except ValidationError as err:
                 errors.append(err)
                 break
-            
+
         # for any remaining segments, deal with them as command / value pairings
         while segments:
             # search for our command
@@ -391,6 +413,20 @@ def parse_sms_submission(self, message_obj):
                 # parse this segment
                 if self.is_command(segment, commands):
                     command = segment
+
+                # check for cases where a fully-alpha command and an integer value
+                # are together with no separator (a common and unambiguous typeo),
+                # e.g. 'age27'
+                match_num = re.search('[0-9]', segment)
+                if match_num:
+                    command_segment = segment[0:match_num.start()]
+                    num_segment = segment[match_num.start():]
+                    possible_command = self.is_command(command_segment, commands)                    
+                    if possible_command and commands[possible_command.lower()].field_type == XFormField.TYPE_INT and num_segment.isdigit():
+                        segments.insert(0, num_segment)
+                        command = command_segment
+
+                if command:
                     break
 
             # no command found?  break out
@@ -420,6 +456,19 @@ def parse_sms_submission(self, message_obj):
                     segments.insert(0, segment)
                     break
 
+                # check for cases where a fully-alpha command and an integer value
+                # are together with no separator (a common and unambiguous typeo),
+                # e.g. 'age27'
+                match_num = re.search('[0-9]', segment)
+                if match_num:
+                    command_segment = segment[0:match_num.start()]
+                    num_segment = segment[match_num.start():]
+                    possible_command = self.is_command(command_segment, commands)                    
+                    if possible_command and commands[possible_command.lower()].field_type == XFormField.TYPE_INT and num_segment.isdigit():
+                        segments.insert(0, num_segment)
+                        segments.insert(0, command_segment)
+                        break
+
                 # this isn't a command, but rather part of the value
                 if not value:
                     value = segment
@@ -528,12 +577,14 @@ def process_sms_submission(self, message_obj):
         """
         message = message_obj.text
         connection = message_obj.connection
-
+        db_message = None
+        if hasattr(message_obj, 'db_message'):
+            db_message = message_obj.db_message
         # parse our submission
         sub_dict = self.parse_sms_submission(message_obj)
 
         # create our new submission, we'll add field values as we parse them
-        submission = XFormSubmission(xform=self, type='sms', raw=message, connection=connection)
+        submission = XFormSubmission(xform=self, type='sms', message=db_message, raw=message, connection=connection)
         submission.save()
 
         # build our template response
@@ -924,11 +975,12 @@ class XFormSubmission(models.Model):
 
     xform = models.ForeignKey(XForm, related_name='submissions')
     type = models.CharField(max_length=8, choices=SUBMISSION_CHOICES)
-    connection = models.ForeignKey(Connection, null=True)
+    connection = models.ForeignKey(Connection, null=True, related_name='submissions')
     raw = models.TextField()
     has_errors = models.BooleanField(default=False)
     created = models.DateTimeField(auto_now_add=True)
     confirmation_id = models.IntegerField(default=0)
+    message = models.ForeignKey(Message, null=True, related_name='submissions')
 
     confirmation_lock = Lock()
 

rapidsms_xforms/tests.py

@@ -695,18 +695,23 @@ def testEpi(self):
         #+epi ma 12, bd 5
 
         xform = XForm.on_site.create(name='epi_test', keyword='epi', owner=self.user, command_prefix=None, 
-                                     keyword_prefix = '+', separator = ',',
+                                     keyword_prefix = '+', separator = ',;:*.\\s"',
                                      site=Site.objects.get_current(), response='thanks')
 
         f1 = xform.fields.create(field_type=XFormField.TYPE_INT, name='ma', command='ma', order=0)        
         f2 = xform.fields.create(field_type=XFormField.TYPE_INT, name='bd', command='bd', order=1)
+        f3 = xform.fields.create(field_type=XFormField.TYPE_INT, name='tb', command='tb', order=2)
+        f4 = xform.fields.create(field_type=XFormField.TYPE_INT, name='yf', command='yf', order=3)
 
-        submission = xform.process_sms_submission(IncomingMessage(None, "+epi ma 12, bd 5"))
+        # huge yellow fever outbreak, from a really poorly-trained texter
+        submission = xform.process_sms_submission(IncomingMessage(None, "+epi MA12, bd 5. tb0;yf314"))
 
         self.failUnlessEqual(submission.has_errors, False)
-        self.failUnlessEqual(len(submission.values.all()), 2)
+        self.failUnlessEqual(len(submission.values.all()), 4)
         self.failUnlessEqual(submission.values.get(attribute__name='ma').value, 12)
         self.failUnlessEqual(submission.values.get(attribute__name='bd').value, 5)
+        self.failUnlessEqual(submission.values.get(attribute__name='tb').value, 0)
+        self.failUnlessEqual(submission.values.get(attribute__name='yf').value, 314)
 
         # missing value
         submission = xform.process_sms_submission(IncomingMessage(None, "+epi ma"))
@@ -716,6 +721,10 @@ def testEpi(self):
         submission = xform.process_sms_submission(IncomingMessage(None, "+epi ma 12, ma 5"))
         self.failUnless(submission.has_errors)
 
+        # zero values
+        submission = xform.process_sms_submission(IncomingMessage(None, "+epi ma 0"))
+        self.failIf(submission.has_errors)
+
         #+muac davey crockett, m, 6 months, red
 
         xform = XForm.on_site.create(name='muac_test', keyword='muac', owner=self.user, command_prefix=None, 

rapidsms_xforms/views.py

@@ -99,9 +99,14 @@ def xforms(req):
 
 
 class NewXFormForm(forms.ModelForm): # pragma: no cover
+    separator = forms.MultipleChoiceField(choices=XForm.SEPARATOR_CHOICES, required=False)
     class Meta:
         model = XForm
-        fields = ('name', 'keyword','keyword_prefix', 'command_prefix', 'separator', 'description', 'response')
+        fields = ('name', 'keyword','keyword_prefix', 'command_prefix', 'description', 'response')
+
+    def clean_separator(self):
+        """ separator should be joined into a string of all chosen separators"""
+        return ''.join(self.cleaned_data['separator'])
 
     helper = FormHelper()
 
@@ -124,10 +129,14 @@ class Meta:
     helper.add_layout(layout)
 
 class EditXFormForm(forms.ModelForm): # pragma: no cover
+    separator = forms.MultipleChoiceField(choices=XForm.SEPARATOR_CHOICES, required=False)
     class Meta:
         model = XForm
-        fields = ('name', 'keyword','keyword_prefix', 'command_prefix', 'separator', 'description', 'response', 'active')
+        fields = ('name', 'keyword','keyword_prefix', 'command_prefix', 'description', 'response', 'active')
 
+    def clean_separator(self):
+        """ separator should be joined into a string of all chosen separators"""
+        return ''.join(self.cleaned_data['separator'])
 
     helper = FormHelper()
 
@@ -163,6 +172,8 @@ def new_xform(req):
             # and the site
             xform.site = Site.objects.get_current()
 
+            # add the separators
+            xform.separator = form.cleaned_data['separator']
             # commit it
             xform.save()
 
@@ -202,11 +213,13 @@ def edit_form(req, form_id):
         form = EditXFormForm(req.POST, instance=xform)
         if form.is_valid():
             xform = form.save()
+            xform.separator = form.cleaned_data['separator']
+            xform.save()
             return render_to_response("xforms/form_details.html", 
                 {"xform" : xform},
                 context_instance=RequestContext(req))
     else:
-        form = EditXFormForm(instance=xform)
+        form = EditXFormForm(instance=xform, initial={'separator':list(xform.separator) if xform.separator else None})
 
     return render_to_response("xforms/form_edit.html", 
         { 'form': form, 'xform': xform, 'fields': fields, 'field_count' : len(fields), 'breadcrumbs' : breadcrumbs },
@@ -232,12 +245,18 @@ class FieldForm(forms.ModelForm):
     def updateTypes(self):
         self.fields['field_type'].widget.choices = [(choice['type'], choice['label']) for choice in XFormField.TYPE_CHOICES.values()]
 
+    def clean_field_type(self):
+        toret = self.cleaned_data['field_type']
+        if self.xform.separator.find('.') >= 0 and XFormField.TYPE_CHOICES[toret]['db_type'] == XFormField.TYPE_FLOAT:
+            raise forms.ValidationError("You cannot have float values along with period (.) separators in an XForm.  Please edit the XForm separators and try again.")
+        return toret
+
     class Meta:
         model = XFormField
         fields = ('field_type', 'name', 'command', 'description')
         widgets = {
             'description': forms.Textarea(attrs={'cols': 30, 'rows': 2}),
-            'field_type': forms.Select()
+            'field_type': forms.Select(),
         }
 
 class ConstraintForm(forms.ModelForm):
@@ -253,6 +272,7 @@ def add_field(req, form_id):
     if req.method == 'POST':
         form = FieldForm(req.POST)
         form.updateTypes()
+        form.xform = xform
         if form.is_valid():
             field = form.save(commit=False)
             field.xform = xform
@@ -365,7 +385,7 @@ def edit_field (req, form_id, field_id):
     if req.method == 'POST':
         form = FieldForm(req.POST, instance=field)
         form.updateTypes()
-
+        form.xform = xform
         if form.is_valid():
             field = form.save(commit=False)
             field.xform = xform