-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparser.py
More file actions
10245 lines (8495 loc) · 435 KB
/
Copy pathparser.py
File metadata and controls
10245 lines (8495 loc) · 435 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Parser for the NexusLang (NexusLang).
This module converts a stream of tokens into an Abstract Syntax Tree (AST).
"""
from typing import Type
from nexuslang.parser.lexer import TokenType, Token
from nexuslang.errors import NxlSyntaxError, suggest_correction
from nexuslang.parser.ast import (
Program, VariableDeclaration, IndexAssignment, MemberAssignment, DereferenceAssignment, FunctionDefinition, Parameter,
IfStatement, WhileLoop, ForLoop, MemoryAllocation, MemoryDeallocation,
ClassDefinition, PropertyDeclaration, MethodDefinition,
ObjectInstantiation, MemberAccess,
ConcurrentExecution, TryCatch, RaiseStatement, BinaryOperation,
UnaryOperation, Literal, Identifier, FunctionCall, PrintStatement, RepeatNTimesLoop, RepeatWhileLoop,
TypeCastExpression,
ReturnStatement, BreakStatement, ContinueStatement, Block, ConcurrentBlock, TryCatchBlock, PanicStatement,
SendStatement, CloseStatement,
# Module-related AST nodes
ImportStatement, SelectiveImport, ModuleAccess, PrivateDeclaration,
InterfaceDefinition, AbstractClassDefinition, TraitDefinition,
TypeAliasDefinition, TypeParameter, TypeConstraint, TypeGuard,
AbstractMethodDefinition,
ListExpression, DictExpression, SliceExpression, IndexExpression,
ListComprehension, DictComprehension, TernaryExpression, NullCoalesceExpression,
LambdaExpression, AsyncExpression, AwaitExpression,
YieldExpression, GeneratorExpression,
# Low-level pointer operations
AddressOfExpression, DereferenceExpression, SizeofExpression, PointerType,
# Smart pointer operations
DowngradeExpression, UpgradeExpression,
# Ownership / borrow operations
MoveExpression, BorrowExpression, DropBorrowStatement,
# Lifetime annotations
LifetimeAnnotation, BorrowExpressionWithLifetime, ParameterWithLifetime, ReturnTypeWithLifetime,
# Allocator hints and parallel execution
AllocatorHint, ParallelForLoop,
# Conditional compilation / platform detection
ConditionalCompilationBlock,
# Struct and union types
StructDefinition, StructField, UnionDefinition, EnumDefinition, EnumMember, OffsetofExpression, TypeCastExpression,
# Inline assembly
InlineAssembly,
# Decorators and Macros
Decorator, MacroDefinition, MacroExpansion,
ComptimeExpression, ComptimeConst, ComptimeAssert, AttributeDeclaration,
# Pattern matching
MatchExpression, MatchCase, Pattern, LiteralPattern, IdentifierPattern,
WildcardPattern, VariantPattern, TuplePattern, ListPattern,
# Switch statement
SwitchStatement, SwitchCase,
# String operations
StringLiteral, FStringExpression,
# Smart pointers and memory management
RcType, WeakType, ArcType, RcCreation,
ChannelCreation, ReceiveExpression,
# Native test framework
TestBlock, DescribeBlock, ItBlock, ParameterizedTestBlock,
BeforeEachBlock, AfterEachBlock,
# Assertion library
ExpectStatement,
# Contract programming
RequireStatement, EnsureStatement, GuaranteeStatement,
InvariantStatement, OldExpression, SpecAnnotation, SpecBlock,
# Higher-kinded type annotations
KindAnnotation, StarKindAnnotation, ArrowKindAnnotation,
)
class Parser:
"""Parses a stream of tokens into an AST."""
# Single-token statement dispatch — maps TokenType → method name.
# Populated at class definition time to avoid repeated dict construction.
_STMT_DISPATCH: dict = {}
def __init__(self, tokens, source=None):
self.tokens = tokens
self.source = source # Store full source for error context
self.current_token_index = 0
self.current_token = tokens[0] if tokens else None
self._in_argument_context = False # Prevents parsing trailing blocks in function arguments
def error(self, message, error_type_key=None):
"""Raise a syntax error with enhanced context and suggestions."""
if self.current_token:
line = self.current_token.line
column = self.current_token.column
token_value = self.current_token.lexeme if self.current_token.lexeme else str(self.current_token.type)
source_line = self.current_token.source_line
# Determine expected and got for better error messages
expected = None
got = str(self.current_token.type)
# Extract expected from message if present
if "Expected" in message:
expected_part = message.split("Expected")[1].split(",")[0].strip()
expected = expected_part
elif message.startswith("Unexpected token"):
expected = "a valid expression or statement"
# Determine error type key from context if not provided
if not error_type_key:
if "Unexpected end" in message or "Expected" in message:
error_type_key = "unexpected_token"
elif "missing" in message.lower():
error_type_key = "missing_end"
elif "Invalid" in message:
error_type_key = "invalid_syntax"
# Get suggestion based on context
suggestion = self._get_error_suggestion(message, token_value)
raise NxlSyntaxError(
message,
line=line,
column=column,
source_line=source_line,
suggestion=suggestion,
expected=expected,
got=got,
error_type_key=error_type_key,
full_source=self.source
)
else:
raise NxlSyntaxError(message, full_source=self.source)
def _get_error_suggestion(self, message, token_value):
"""Get a helpful suggestion based on the error message."""
message_lower = message.lower()
token_str = str(token_value).lower()
# Common mistakes and suggestions
if "expected 'end'" in message_lower or "expected end" in message_lower:
return "Make sure to close all blocks (if, while, for, function, etc.) with 'end'"
elif "expected TokenType.END" in message:
return "Did you forget to add 'end' to close a block (if/while/for/function/try)?"
elif "unexpected" in message_lower and "indent" in message_lower:
return "Check your indentation - NexusLang uses consistent indentation for blocks"
elif "expected" in message_lower and "got" in message_lower:
# Extract what was expected
if "TokenType." in message:
return "Check the syntax - you might be missing a keyword or punctuation"
elif "undefined" in message_lower or "not defined" in message_lower:
return "Make sure you've declared this variable with 'set' before using it"
return None
def eat(self, token_type):
"""
Consume the current token if it matches the expected type.
Otherwise, raise an error.
"""
if self.current_token.type == token_type:
token = self.current_token
self.advance()
return token
else:
self.error(f"Expected {token_type}, got {self.current_token.type}")
def consume(self, token_type, error_message=None):
"""
Consume the current token if it matches the expected type.
Otherwise, raise an error with custom message.
Alias for eat() with optional custom error message.
"""
if self.current_token.type == token_type:
token = self.current_token
self.advance()
return token
else:
if error_message:
self.error(error_message)
else:
self.error(f"Expected {token_type}, got {self.current_token.type}")
def advance(self):
"""Advance to the next token."""
self.current_token_index += 1
if self.current_token_index < len(self.tokens):
self.current_token = self.tokens[self.current_token_index]
else:
self.current_token = None
def previous(self):
"""Return the previous token (the one we just consumed)."""
if self.current_token_index > 0:
return self.tokens[self.current_token_index - 1]
return None
def check(self, token_type):
"""Check if current token is of given type without consuming it."""
if self.current_token is None:
return False
return self.current_token.type == token_type
def is_at_end(self):
"""Check if we've reached the end of tokens."""
return self.current_token is None or self.current_token.type == TokenType.EOF
def match(self, *token_types):
"""
Check if current token matches any of the given types.
If so, consume it and return True.
"""
for token_type in token_types:
if self.check(token_type):
self.advance()
return True
return False
def peek(self, n=1):
"""Look ahead n tokens without advancing."""
peek_index = self.current_token_index + n
if peek_index < len(self.tokens):
return self.tokens[peek_index]
return None
def _skip_whitespace_tokens(self):
"""Skip NEWLINE, INDENT, DEDENT, and DOC_COMMENT boundary tokens."""
while (self.current_token and
self.current_token.type in (
TokenType.NEWLINE, TokenType.INDENT, TokenType.DEDENT,
TokenType.DOC_COMMENT,
)):
self.advance()
def parse(self):
"""Parse the token stream and return the AST."""
return self.program()
def program(self):
"""Parse a program."""
statements = []
# Collects consecutive ## doc-comment lines preceding a definition.
_pending_doc_lines: list = []
# Node types that represent named definitions (eligible for doc attachment).
_DOC_TARGET_TYPES = (
"function_definition", "async_function_definition",
"class_definition", "struct_definition",
"enum_definition", "trait_definition", "interface_definition",
"module_definition",
)
while self.current_token and self.current_token.type != TokenType.EOF:
# Skip NEWLINE/INDENT/DEDENT boundary tokens and empty-lexeme identifiers
if self.current_token.type in (TokenType.NEWLINE, TokenType.INDENT, TokenType.DEDENT):
self.advance()
continue
# Collect documentation comments
if self.current_token.type == TokenType.DOC_COMMENT:
_pending_doc_lines.append(self.current_token.lexeme)
self.advance()
continue
if self.current_token.type == TokenType.IDENTIFIER and self.current_token.lexeme.strip() == '':
self.advance()
continue
# Parse statement
try:
statement = self.statement()
if statement:
# Attach any buffered doc comment to definition nodes
if _pending_doc_lines and getattr(statement, 'node_type', None) in _DOC_TARGET_TYPES:
statement.doc = "\n".join(_pending_doc_lines)
statements.append(statement)
except SyntaxError as e:
raise
finally:
# Always clear the pending doc after consuming a statement
_pending_doc_lines = []
# Skip empty lines and whitespace after statement
while self.current_token and self.current_token.type == TokenType.IDENTIFIER and self.current_token.lexeme.strip() == '':
self.advance()
return Program(statements)
def statement(self):
"""Parse a statement."""
if not self.current_token:
return None
token = self.current_token
try:
# Fast dispatch for simple single-token statements
_handler = self._STMT_DISPATCH.get(token.type)
if _handler:
return getattr(self, _handler)()
# Handle special multi-token or context-dependent statements
return self._handle_special_statements(token)
except SyntaxError as e:
# Attempt error recovery
self.error_recovery()
return None
def _handle_special_statements(self, token):
"""Handle special statements that require lookahead or context."""
if token.type == TokenType.LABEL:
return self.labeled_loop_statement()
elif token.type == TokenType.FOR_EACH:
return self.for_loop()
elif token.type == TokenType.PARALLEL:
return self.parse_parallel_for()
elif token.type == TokenType.WHEN:
return self._handle_when_statement()
elif token.type == TokenType.REPEAT:
return self.for_loop()
elif token.type == TokenType.RUN:
return self.concurrent_execution()
elif token.type in (TokenType.ADD, TokenType.APPEND):
return self.add_statement()
elif token.type == TokenType.SEND:
return self.parse_send_statement()
elif token.type == TokenType.CREATE:
return self.create_statement()
elif token.type == TokenType.FUNCTION:
return self.function_definition_short()
elif token.type == TokenType.ASYNC:
return self.async_function_definition()
elif token.type == TokenType.IDENTIFIER:
return self._handle_identifier_statement(token)
elif token.type == TokenType.PACKED:
return self._handle_packed_struct()
elif token.type in (TokenType.RETURN, TokenType.RETURNS):
return self.return_statement()
elif token.type in (TokenType.EXTERN, TokenType.FOREIGN):
return self.extern_declaration()
elif token.type == TokenType.EOF:
return None
elif token.type in (TokenType.INDENT, TokenType.DEDENT, TokenType.NEWLINE):
self.advance()
return None
elif token.type == TokenType.DEFINE:
return self.define_statement()
elif token.type == TokenType.AT:
return self._handle_decorator()
else:
# Default: try to parse as expression statement
expr = self.expression()
return expr if expr else None
def _handle_when_statement(self):
"""Handle 'when' statements - disambiguate conditional compilation vs expression."""
next_tok = self._peek_next()
if next_tok and next_tok.lexeme in ("target", "feature"):
return self.parse_conditional_compilation()
expr = self.expression()
return expr if expr else None
def _handle_identifier_statement(self, token):
"""Handle identifier-based statements (abstract class, expressions)."""
if (token.lexeme.lower() == "abstract" and
self.peek() and self.peek().type == TokenType.CLASS):
return self.abstract_class_short_syntax()
else:
expr = self.expression()
return expr if expr else None
def _handle_packed_struct(self):
"""Handle 'packed struct' declarations."""
self.advance() # consume 'packed'
if self.current_token and self.current_token.type == TokenType.STRUCT:
return self.struct_definition(packed=True)
else:
self.error("Expected 'struct' after 'packed'")
def _handle_decorator(self):
"""Handle decorator statements - collect and apply to next function/class."""
decorators = []
while self.current_token and self.current_token.type == TokenType.AT:
decorators.append(self.parse_decorator())
while self.current_token and self.current_token.type == TokenType.NEWLINE:
self.advance()
while self.current_token and self.current_token.type == TokenType.NEWLINE:
self.advance()
if self.current_token and self.current_token.type == TokenType.FUNCTION:
func_def = self.function_definition_short()
func_def.decorators = decorators
return func_def
elif self.current_token and self.current_token.type == TokenType.CLASS:
class_def = self.class_definition()
class_def.decorators = decorators
return class_def
else:
self.error("Decorators can only be applied to functions or classes")
def define_statement(self):
"""Dispatch DEFINE statements to the appropriate parser based on lookahead."""
# DEFINE can start: function, class, interface, trait, method, etc.
# Look ahead to determine which construct this is
# Peek ahead to see what comes after DEFINE [A]
lookahead_index = 1
next_token = self.peek(lookahead_index)
# Skip optional 'a' or 'an'
if next_token and next_token.type == TokenType.A:
lookahead_index += 1
next_token = self.peek(lookahead_index)
elif next_token and next_token.type == TokenType.AN:
lookahead_index += 1
next_token = self.peek(lookahead_index)
elif next_token and next_token.type == TokenType.IDENTIFIER and next_token.lexeme.lower() in ['an']:
lookahead_index += 1
next_token = self.peek(lookahead_index)
# Determine construct type from next significant token
if not next_token:
self.error("Unexpected end of file after DEFINE")
if next_token.type == TokenType.FUNCTION:
return self.function_definition()
elif next_token.type == TokenType.CLASS:
return self.class_definition()
elif next_token.type == TokenType.INTERFACE:
return self.interface_definition()
elif next_token.type == TokenType.TRAIT:
return self.trait_definition()
elif next_token.type == TokenType.IDENTIFIER:
# Could be "method", "property", etc.
if next_token.lexeme.lower() == 'method':
# This is likely inside a class definition
# For now, error - method definitions should be inside class bodies
self.error("Method definitions must be inside class bodies")
else:
self.error(f"Unexpected identifier '{next_token.lexeme}' after DEFINE")
else:
self.error(f"Unexpected token {next_token.type} after DEFINE")
def variable_declaration(self):
"""Parse a variable declaration or assignment.
Grammar:
SET identifier TO expression
SET object.property TO expression
SET array[index] TO expression
SET (value at pointer) TO expression
"""
# Consume SET token
if self.current_token.type != TokenType.SET:
self.error(f"Expected SET, got {self.current_token.type}")
self.advance() # consume SET
# Check for dereference assignment: set (value at ptr) to value
# Only handle if we see both ( and dereference token
if (self.current_token and
self.current_token.type == TokenType.LEFT_PAREN and
self.peek() and
self.peek().type == TokenType.DEREFERENCE):
self.advance() # consume (
self.advance() # consume DEREFERENCE (value at)
# Parse pointer expression
pointer_expr = self.expression()
# Expect closing paren
if self.current_token.type != TokenType.RIGHT_PAREN:
self.error(f"Expected ) after dereference expression, got {self.current_token.type}")
self.advance() # consume )
# Expect TO
if self.current_token.type != TokenType.TO:
self.error(f"Expected TO after dereference target, got {self.current_token.type}")
self.advance() # consume TO
# Parse value expression
value = self.expression()
if value is None:
self.error("Expected a value expression after TO")
# Create DereferenceExpression as target
deref_expr = DereferenceExpression(pointer_expr)
return DereferenceAssignment(deref_expr, value)
# Parse the left-hand side (can be identifier or member access)
# We need to parse this as a primary expression to handle member access
lhs_start_token = self.current_token
# Get base identifier
if self.current_token.type == TokenType.IDENTIFIER:
var_name = self.current_token.lexeme
elif hasattr(self.current_token, 'lexeme') and self.current_token.lexeme:
# Allow keywords to be used as variable names in this context
var_name = self.current_token.lexeme
else:
self.error("Expected an identifier after SET")
line_num = self.current_token.line if hasattr(self.current_token, 'line') else 0
self.advance() # consume variable name
# Check if this is member access (object.property) or index access (array[index])
if self.current_token and self.current_token.type == TokenType.DOT:
# Parse member access chain
base = Identifier(var_name)
lhs = self._parse_member_access(base)
# Check for TO
if self.current_token.type != TokenType.TO:
self.error(f"Expected TO after member access, got {self.current_token.type}")
self.advance() # consume TO
# Parse the value expression
value = self.expression()
if value is None:
self.error("Expected a value expression after TO")
# Return a member assignment node
return MemberAssignment(lhs, value)
elif self.current_token and self.current_token.type == TokenType.LEFT_BRACKET:
# Parse index access: set array[0] to value OR set dict["key"] to value
# OR set array[0].field to value (member access on indexed element)
base = Identifier(var_name)
lhs = self._parse_index_access(base)
# Check for TO
if self.current_token.type != TokenType.TO:
self.error(f"Expected TO after index access, got {self.current_token.type}")
self.advance() # consume TO
# Parse the value expression
value = self.expression()
if value is None:
self.error("Expected a value expression after TO")
# Check if lhs is MemberAccess (e.g., array[0].x) or IndexExpression (e.g., array[0])
if lhs.__class__.__name__ == 'MemberAccess':
# This is a member assignment (e.g., array[0].x = 5)
return MemberAssignment(lhs, value)
else:
# This is an index assignment (e.g., array[0] = value)
return IndexAssignment(lhs, value)
else:
# Simple variable assignment
# Check for TO
if self.current_token.type != TokenType.TO:
self.error(f"Expected TO after variable name, got {self.current_token.type}")
self.advance() # consume TO
# Parse the value expression
value = self.expression()
if value is None:
self.error("Expected a value expression after TO")
# Optional type annotation: set x to value as List of Integer [with allocator arena]
type_annotation = None
if self.current_token and self.current_token.type == TokenType.AS:
self.advance() # consume 'as'
type_annotation = self.parse_type()
# Optional allocator hint: ... with allocator <name>
allocator_name = None
if (self.current_token and self.current_token.type == TokenType.WITH
and self.peek() and self.peek().type == TokenType.ALLOCATOR):
self.advance() # consume 'with'
self.advance() # consume 'allocator'
if self.current_token and self.current_token.type in (
TokenType.IDENTIFIER, TokenType.ALLOCATOR):
allocator_name = self.current_token.lexeme
self.advance() # consume allocator name
return VariableDeclaration(var_name, value, type_annotation, allocator_name)
def add_statement(self):
"""Parse an add/append statement.
Grammar:
ADD expression TO identifier
APPEND expression TO identifier
This is a shorthand for appending to a list or collection.
"""
# Consume ADD or APPEND token
if self.current_token.type not in (TokenType.ADD, TokenType.APPEND):
self.error(f"Expected ADD or APPEND, got {self.current_token.type}")
self.advance() # consume ADD or APPEND
# Parse the value to add
value = self.expression()
if value is None:
self.error("Expected an expression after ADD")
# Expect TO keyword
if not self.current_token or self.current_token.type != TokenType.TO:
self.error(f"Expected TO in add statement, got {self.current_token.type if self.current_token else 'EOF'}")
self.advance() # consume TO
# Parse the target variable (list/collection or member access like "this.grades")
target = self.comparison() # Use comparison to handle member access
line_num = self.current_token.line if hasattr(self.current_token, 'line') else 0
# Create a function call to list_append
# This translates "add X to Y" into "list_append(Y, X)"
return FunctionCall("list_append", [target, value], [], line_number=line_num)
def create_statement(self):
"""Parse a create statement for variable initialization.
Handles both the classic form and structured type-definition forms:
CREATE identifier AS expression
CREATE member_access AS expression
CREATE (A|AN) abstract CLASS CALLED name WITH: body
CREATE (A|AN) CLASS CALLED name WITH A GENERIC TYPE PARAMETER T THAT EXTENDS bound
CREATE (A|AN) TRAIT CALLED name WITH: body
CREATE (A|AN) TYPE alias CALLED name THAT IS A ...
CREATE (A|AN) <type_keyword> CALLED name [AND SET it TO expr]
"""
line_number = self.current_token.line
self.advance() # consume CREATE
# Detect article-prefixed class/trait/type constructs when next token is A or AN
if self.current_token and self.current_token.type in (TokenType.A, TokenType.AN):
self.advance() # consume A/AN
tok = self.current_token
# CREATE (A|AN) IDENTIFIER("abstract") CLASS CALLED name WITH: body
if tok and tok.type == TokenType.IDENTIFIER and tok.lexeme.lower() == 'abstract':
self.advance() # consume "abstract"
if self.current_token and self.current_token.type == TokenType.CLASS:
self.advance() # consume CLASS
return self._parse_abstract_class_def(line_number)
self.error(f"Expected CLASS after 'abstract', got {self.current_token.type if self.current_token else 'EOF'}")
# CREATE (A|AN) CLASS CALLED name WITH A GENERIC TYPE PARAMETER T THAT EXTENDS bound
elif tok and tok.type == TokenType.CLASS:
self.advance() # consume CLASS
return self._parse_generic_class_def(line_number)
# CREATE (A|AN) TRAIT CALLED name WITH: body
elif tok and tok.type == TokenType.TRAIT:
self.advance() # consume TRAIT
return self._parse_trait_def(line_number)
# CREATE (A|AN) TYPE [alias] CALLED name THAT IS ...
elif tok and tok.type == TokenType.TYPE:
self.advance() # consume TYPE
if self.current_token and self.current_token.type == TokenType.IDENTIFIER and \
self.current_token.lexeme.lower() == 'alias':
self.advance() # consume "alias"
return self._parse_type_alias_def(line_number)
# CREATE (A|AN) <type_keyword> CALLED name [AND SET it TO expr]
# e.g., "Create an integer called length and set it to the length of x."
elif tok and tok.type in (TokenType.INTEGER, TokenType.FLOAT, TokenType.STRING,
TokenType.BOOLEAN, TokenType.LIST, TokenType.DICTIONARY,
TokenType.NUMBER, TokenType.OBJECT, TokenType.LENGTH,
TokenType.IDENTIFIER):
type_name = tok.lexeme.lower() if tok else None
self.advance() # consume type keyword
# Consume optional CALLED
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
# Get variable name (any token type - e.g. LENGTH keyword as "length")
var_name = self.current_token.lexeme if self.current_token else "unknown"
self.advance()
# Parse optional "and set it/varname to expression"
init_value = None
if self.current_token and self.current_token.type == TokenType.AND:
self.advance() # consume AND
if self.current_token and self.current_token.type == TokenType.SET:
self.advance() # consume SET
# Skip "it" or the variable reference (anything that isn't TO)
if self.current_token and self.current_token.type != TokenType.TO:
self.advance()
# Consume TO
if self.current_token and self.current_token.type == TokenType.TO:
self.advance()
try:
init_value = self.expression()
except Exception:
init_value = None
# Skip to end of statement
while self.current_token and self.current_token.type not in (
TokenType.DOT, TokenType.EOF, TokenType.DEDENT):
self.advance()
# Consume optional DOT
if self.current_token and self.current_token.type == TokenType.DOT:
self.advance()
return VariableDeclaration(var_name, init_value, type_name)
else:
self.error(
f"Unexpected token in create statement after 'a/an': "
f"{tok.type if tok else 'EOF'}"
)
# Original logic: CREATE target AS expression
target = self.comparison()
if not self.current_token or self.current_token.type != TokenType.AS:
self.error(
f"Expected AS in create statement, got "
f"{self.current_token.type if self.current_token else 'EOF'}"
)
self.advance() # consume AS
value = self.expression()
if value is None:
self.error("Expected an expression after AS")
if target.__class__.__name__ == 'Identifier':
return VariableDeclaration(target.name, value, None)
elif target.__class__.__name__ == 'MemberAccess':
from ..parser.ast import MemberAssignment
return MemberAssignment(target, value)
else:
return VariableDeclaration(str(target), value, None)
def parse_send_statement(self):
"""Parse a send statement: send <value> to <channel>."""
line_number = self.current_token.line
self.eat(TokenType.SEND)
value = self.expression()
self.eat(TokenType.TO)
channel = self.expression()
return SendStatement(value, channel, line_number)
def parse_close_statement(self):
"""Parse a close statement: close [with] <channel>."""
line_number = self.current_token.line
self.eat(TokenType.CLOSE)
if self.current_token and self.current_token.type == TokenType.WITH:
self.advance()
channel = self.expression()
return CloseStatement(channel, line_number)
# ------------------------------------------------------------------
# Helper parsers for structured type constructs
# ------------------------------------------------------------------
def _parse_abstract_class_def(self, line_number):
"""Parse: CALLED name WITH: [INDENT methods DEDENT]"""
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
class_name = self.current_token.lexeme if self.current_token else "Unknown"
self.advance()
if self.current_token and self.current_token.type == TokenType.WITH:
self.advance()
if self.current_token and self.current_token.type == TokenType.COLON:
self.advance()
# Skip NEWLINE before INDENT
while self.current_token and self.current_token.type == TokenType.NEWLINE:
self.advance()
if self.current_token and self.current_token.type == TokenType.INDENT:
self.advance()
abstract_methods = []
concrete_methods = []
while self.current_token and self.current_token.type not in (TokenType.DEDENT, TokenType.EOF):
# Skip NEWLINE tokens between method definitions
if self.current_token.type == TokenType.NEWLINE:
self.advance()
continue
# Skip A/AN
if self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
if not self.current_token or self.current_token.type in (TokenType.DEDENT, TokenType.EOF):
break
# Read modifier: "abstract" or "concrete"
modifier = None
if self.current_token and self.current_token.type == TokenType.IDENTIFIER:
modifier = self.current_token.lexeme.lower()
self.advance()
# Eat METHOD
if self.current_token and self.current_token.type == TokenType.METHOD:
self.advance()
# Eat CALLED
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
# Get method name (any token, e.g. EQUAL_TO for "equals")
method_name = self.current_token.lexeme if self.current_token else "unknown"
self.advance()
# Skip until RETURNS or DOT
while self.current_token and self.current_token.type not in (
TokenType.RETURNS, TokenType.DOT, TokenType.EOF, TokenType.DEDENT):
self.advance()
return_type = None
if self.current_token and self.current_token.type == TokenType.RETURNS:
self.advance()
if self.current_token and self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
return_type = self.current_token.lexeme.lower() if self.current_token else None
if self.current_token and self.current_token.type != TokenType.DOT:
self.advance()
if self.current_token and self.current_token.type == TokenType.DOT:
self.advance()
if modifier == 'abstract':
abstract_methods.append(
AbstractMethodDefinition(method_name, [], return_type, line_number))
else:
concrete_methods.append(
MethodDefinition(method_name, [], return_type=return_type, line_number=line_number))
if self.current_token and self.current_token.type == TokenType.DEDENT:
self.advance()
if self.current_token and self.current_token.type == TokenType.DEDENT:
self.advance()
return AbstractClassDefinition(
class_name, abstract_methods, concrete_methods, line_number=line_number)
def _parse_generic_class_def(self, line_number):
"""Parse: CALLED name WITH A GENERIC TYPE PARAMETER T THAT EXTENDS bound DOT"""
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
class_name = self.current_token.lexeme if self.current_token else "Unknown"
self.advance()
generic_params = []
if self.current_token and self.current_token.type == TokenType.WITH:
self.advance()
# Consume A/AN
if self.current_token and self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
# Consume GENERIC
if self.current_token and self.current_token.type == TokenType.GENERIC:
self.advance()
# Consume TYPE
if self.current_token and self.current_token.type == TokenType.TYPE:
self.advance()
# Skip IDENTIFIER("parameter")
if self.current_token and self.current_token.type == TokenType.IDENTIFIER and \
self.current_token.lexeme.lower() == 'parameter':
self.advance()
# Get type parameter name (e.g., T or N)
param_name = self.current_token.lexeme if self.current_token else "T"
self.advance()
bounds = []
if self.current_token and self.current_token.type == TokenType.THAT:
self.advance()
if self.current_token and self.current_token.type == TokenType.EXTENDS:
self.advance()
bound_name = self.current_token.lexeme if self.current_token else "Object"
self.advance()
bounds.append(bound_name)
generic_params.append(TypeParameter(param_name, bounds=bounds, line_number=line_number))
if self.current_token and self.current_token.type == TokenType.DOT:
self.advance()
return ClassDefinition(class_name, generic_parameters=generic_params, line_number=line_number)
def _parse_trait_def(self, line_number):
"""Parse: CALLED name WITH: [INDENT required/provided methods DEDENT]"""
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
trait_name = self.current_token.lexeme if self.current_token else "Unknown"
self.advance()
if self.current_token and self.current_token.type == TokenType.WITH:
self.advance()
if self.current_token and self.current_token.type == TokenType.COLON:
self.advance()
# Skip NEWLINE before INDENT
while self.current_token and self.current_token.type == TokenType.NEWLINE:
self.advance()
if self.current_token and self.current_token.type == TokenType.INDENT:
self.advance()
required_methods = []
provided_methods = []
while self.current_token and self.current_token.type not in (TokenType.DEDENT, TokenType.EOF):
# Skip NEWLINE tokens between method definitions
if self.current_token.type == TokenType.NEWLINE:
self.advance()
continue
# Skip A/AN
if self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
if not self.current_token or self.current_token.type in (TokenType.DEDENT, TokenType.EOF):
break
# Read modifier: "required" or "provided"
modifier = None
if self.current_token and self.current_token.type == TokenType.IDENTIFIER:
modifier = self.current_token.lexeme.lower()
self.advance()
# Eat METHOD
if self.current_token and self.current_token.type == TokenType.METHOD:
self.advance()
# Eat CALLED
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
# Get method name (any token, e.g. EQUAL_TO has lexeme "equals")
method_name = self.current_token.lexeme if self.current_token else "unknown"
self.advance()
# Parse optional "THAT TAKES <params> AND RETURNS <type>"
params = []
if self.current_token and self.current_token.type == TokenType.THAT:
self.advance()
if self.current_token and self.current_token.type == TokenType.TAKES:
self.advance()
# Collect parameter types until AND or RETURNS or DOT
while self.current_token and self.current_token.type not in (
TokenType.AND, TokenType.RETURNS, TokenType.DOT, TokenType.EOF,
TokenType.DEDENT):
plex = self.current_token.lexeme
self.advance()
if plex.lower() != 'another':
params.append(
Parameter(f"param_{len(params)}", plex, line_number))
if self.current_token and self.current_token.type == TokenType.AND:
self.advance()
return_type = None
if self.current_token and self.current_token.type == TokenType.RETURNS:
self.advance()
if self.current_token and self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
return_type = self.current_token.lexeme.lower() if self.current_token else None
if self.current_token and self.current_token.type not in (
TokenType.DOT, TokenType.EOF, TokenType.DEDENT):
self.advance()
if self.current_token and self.current_token.type == TokenType.DOT:
self.advance()
mdef = AbstractMethodDefinition(method_name, params, return_type, line_number)
if modifier == 'required':
required_methods.append(mdef)
else:
provided_methods.append(mdef)
if self.current_token and self.current_token.type == TokenType.DEDENT:
self.advance()
return TraitDefinition(
trait_name,
required_methods=required_methods,
provided_methods=provided_methods,
line_number=line_number)
def _parse_type_alias_def(self, line_number):
"""Parse: CALLED name THAT IS A DICTIONARY/LIST ..."""
if self.current_token and self.current_token.type == TokenType.CALLED:
self.advance()
alias_name = self.current_token.lexeme if self.current_token else "Unknown"
self.advance()
target_type = None
if self.current_token and self.current_token.type == TokenType.THAT:
self.advance()
if self.current_token and self.current_token.type == TokenType.IS:
self.advance()
if self.current_token and self.current_token.type in (TokenType.A, TokenType.AN):
self.advance()
if self.current_token and self.current_token.type == TokenType.DICTIONARY:
self.advance()
key_type = "string"
value_type = "string"
if self.current_token and self.current_token.type == TokenType.WITH:
self.advance()
if self.current_token and self.current_token.type in (
TokenType.STRING, TokenType.INTEGER, TokenType.FLOAT, TokenType.BOOLEAN):
key_type = self.current_token.lexeme.lower()
self.advance()
if self.current_token and self.current_token.type == TokenType.IDENTIFIER:
self.advance() # skip "keys"
if self.current_token and self.current_token.type == TokenType.AND:
self.advance()
if self.current_token and self.current_token.type in (
TokenType.STRING, TokenType.INTEGER, TokenType.FLOAT, TokenType.BOOLEAN):
value_type = self.current_token.lexeme.lower()
self.advance()
if self.current_token and self.current_token.type == TokenType.IDENTIFIER:
self.advance() # skip "values"
target_type = f"dictionary<{key_type}, {value_type}>"