Skip to content

Commit 50255e6

Browse files
eiglesias34eiglesias34
committed
Updating compliance
1 parent fb30cf4 commit 50255e6

7 files changed

Lines changed: 2636 additions & 1170 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ You can easily customize your own configurations from the set of features that S
8484

8585
## Version
8686
```
87-
4.7.5.13.5
87+
4.7.5.14
8888
```
8989

9090
## RML-Test Cases

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
4.7.5.13.5
1+
4.7.5.14

‎rdfizer/rdfizer/__init__.py‎

Lines changed: 1242 additions & 569 deletions
Large diffs are not rendered by default.

‎rdfizer/rdfizer/fnml_functions.py‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@
2121
"array_join":"","controls_if":"","string_md5":"","string_contains":"",
2222
"slugify":"","trueCondition":"","isNull":"",
2323
"notEqual":"","equal":"","normalizeDateTime":"","normalizeDate":"",
24-
"listContainsElement":""}
24+
"listContainsElement":"","alwaysReturnsABC":"","string_length":""}
2525

2626

2727
## Define your functions here following examples below, the column "names" from the csv files
2828
## that you aim to use as the input parameters of functions are only required to be provided
2929
## as the keys of "global_dic"
30+
def alwaysReturnsABC():
31+
return "ABC"
32+
3033
def listContainsElement():
3134
if str(global_dic["str"]) in global_dic["list"]:
3235
return True
@@ -44,10 +47,16 @@ def normalizeDateTime():
4447
return str(datetime.strptime(str(global_dic["strDate"]), str(global_dic["pattern"])))
4548

4649
def equal():
47-
if str(global_dic["valueParameter"]) == str(global_dic["valueParameter2"]):
48-
return True
50+
if "valueParam" in global_dic:
51+
if str(global_dic["valueParam"]) == str(global_dic["valueParam2"]):
52+
return True
53+
else:
54+
return False
4955
else:
50-
return False
56+
if str(global_dic["valueParameter"]) == str(global_dic["valueParameter2"]):
57+
return True
58+
else:
59+
return False
5160

5261
def notEqual():
5362
if str(global_dic["valueParameter"]) != str(global_dic["valueParameter2"]):
@@ -75,7 +84,10 @@ def slugify():
7584
return slugify(str(global_dic["str"]))
7685

7786
def string_replace():
78-
return str(global_dic["valueParameter"]).replace(str(global_dic["p_string_find"]),str(global_dic["p_string_replace"]))
87+
if "valueParam" in global_dic:
88+
return str(global_dic["valueParam"]).replace(str(global_dic["param_find"]),str(global_dic["param_replace"]))
89+
else:
90+
return str(global_dic["valueParameter"]).replace(str(global_dic["p_string_find"]),str(global_dic["p_string_replace"]))
7991

8092
def string_contains():
8193
if str(global_dic["string_sub"]) in str(global_dic["valueParameter"]):
@@ -113,11 +125,22 @@ def array_join():
113125
output += str(global_dic["p_string_sep"])
114126
return output
115127

116-
def string_substring():
117-
if int(global_dic["param_int_i_from"]) > len(str(global_dic["valueParameter"])) or int(global_dic["param_int_i_opt_to"]) > len(str(global_dic["valueParameter"])):
118-
return None
119-
else:
120-
return str(global_dic["valueParameter"])[int(global_dic["param_int_i_from"]):int(global_dic["param_int_i_opt_to"])]
128+
def string_substring():
129+
if "p_int_i_from" in global_dic:
130+
if int(global_dic["p_int_i_from"]) == len(str(global_dic["valueParam"])):
131+
return ""
132+
elif int(global_dic["p_int_i_from"]) > len(str(global_dic["valueParam"])):
133+
return None
134+
else:
135+
return str(global_dic["valueParam"])[int(global_dic["p_int_i_from"]):]
136+
else:
137+
if int(global_dic["param_int_i_from"]) >= len(str(global_dic["valueParameter"])) or int(global_dic["param_int_i_opt_to"]) >= len(str(global_dic["valueParameter"])):
138+
return None
139+
else:
140+
return str(global_dic["valueParameter"])[int(global_dic["param_int_i_from"]):int(global_dic["param_int_i_opt_to"])]
141+
142+
def string_length():
143+
return str(len(str(global_dic["valueParam"])))
121144

122145
def length():
123146
return str(len(str(global_dic["valueParam"])))

‎rdfizer/rdfizer/functions.py‎

Lines changed: 92 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1027,34 +1027,63 @@ def base36encode(number, alphabet='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'):
10271027

10281028
def sublist(part_list, full_list):
10291029
for part in part_list:
1030-
if "$." in part:
1031-
if part[2:] not in full_list:
1032-
return False
1030+
if isinstance(part,dict):
1031+
if part["type"] == "template":
1032+
template_references = re.finditer("{(.+?)}", part["value"])
1033+
for reference_match in template_references:
1034+
if "$." in reference_match.group(1):
1035+
if reference_match.group(1)[2:].split(".")[0] not in full_list:
1036+
return False
1037+
else:
1038+
if reference_match.group(1) not in full_list:
1039+
return False
10331040
else:
1034-
if part not in full_list:
1035-
return False
1041+
if "$." in part:
1042+
if part[2:] not in full_list:
1043+
return False
1044+
else:
1045+
if part not in full_list:
1046+
return False
10361047
return True
10371048

10381049
def child_list(childs):
10391050
value = ""
10401051
for child in childs:
1041-
if child not in value:
1042-
value += child + "_"
1052+
if isinstance(child,dict):
1053+
if child["value"] not in value:
1054+
value += child["value"] + "_"
1055+
else:
1056+
if child not in value:
1057+
value += child + "_"
10431058
return value[:-1]
10441059

10451060
def child_list_value(childs,row):
10461061
value = ""
10471062
v = []
10481063
for child in childs:
10491064
if child not in v:
1050-
if "$." in child:
1051-
if row[child[2:]] != None:
1052-
value += str(row[child[2:]]) + "_"
1053-
v.append(child[2:])
1065+
if isinstance(child,dict):
1066+
if child["type"] == "template":
1067+
if "$." in child["value"]:
1068+
inter_value = string_substitution_json(child["value"], "{(.+?)}",row, "object", "yes", "")
1069+
else:
1070+
inter_value = string_substitution(child["value"], "{(.+?)}",row, "object", "yes", "")
1071+
if inter_value != None:
1072+
value += inter_value + "_"
1073+
v.append(child["value"])
1074+
else:
1075+
value += child["value"] + "_"
1076+
v.append(child["value"])
10541077
else:
1055-
if row[child] != None:
1056-
value += str(row[child]) + "_"
1057-
v.append(child)
1078+
if "$." in child:
1079+
inter_value = string_substitution_json(child, ".+",row, "object", "yes", "")
1080+
if inter_value != None:
1081+
value += inter_value[1:-1] + "_"
1082+
v.append(child[2:])
1083+
else:
1084+
if row[child] != None:
1085+
value += str(row[child]) + "_"
1086+
v.append(child)
10581087
return value[:-1]
10591088

10601089
def child_list_value_array(childs,row,row_headers):
@@ -1095,6 +1124,45 @@ def string_substitution_json(string, pattern, row, term, ignore, iterator):
10951124
match = temp_match[2:-2]
10961125
else:
10971126
match = reference_match.group(1)[2:]
1127+
elif "$" == reference_match.group(1)[:1]:
1128+
if "['\\{" in reference_match.group(1) or "[\"\\{" in reference_match.group(1):
1129+
match = reference_match.group(1).split("[")[1]
1130+
match = match.split("]")[0]
1131+
match = match.replace("\\","")
1132+
match = match.replace("{","")
1133+
match = match.replace("}","")
1134+
match = match.replace("'","")
1135+
match = match.replace("\"","")
1136+
match = "{" + match + "}"
1137+
elif "[" in reference_match.group(1)[1:]:
1138+
match = reference_match.group(1)[3:-2]
1139+
else:
1140+
match = reference_match.group(1)[2:]
1141+
elif "\\{" in reference_match.group(1):
1142+
if "['\\{" in reference_match.group(1) or "[\"\\{" in reference_match.group(1):
1143+
match = reference_match.group(1).split("[")[1]
1144+
match = match.split("]")[0]
1145+
match = match.replace("\\","")
1146+
match = match.replace("{","")
1147+
match = match.replace("}","")
1148+
match = match.replace("'","")
1149+
match = match.replace("\"","")
1150+
match = "{" + match + "}"
1151+
else:
1152+
match = reference_match.group(1).replace("\\{","").replace("\\}","")
1153+
match = match.replace("{","").replace("}","")
1154+
if match[0] == " ":
1155+
match = match[1:]
1156+
if "$." == match[:2]:
1157+
if "[" in match[1:]:
1158+
match = match[3:-2]
1159+
else:
1160+
match = match[2:]
1161+
elif "$" == match[:1]:
1162+
if "[" in match[1:]:
1163+
match = match[3:-2]
1164+
else:
1165+
match = match[2:]
10981166
else:
10991167
match = reference_match.group(1)
11001168
else:
@@ -1103,6 +1171,11 @@ def string_substitution_json(string, pattern, row, term, ignore, iterator):
11031171
match = reference_match.group(1)[2:]
11041172
else:
11051173
match = reference_match.group(1)[2:]
1174+
elif "$" == reference_match.group(1)[:1]:
1175+
if "[" in reference_match.group(1)[2:]:
1176+
match = reference_match.group(1)[2:]
1177+
else:
1178+
match = reference_match.group(1)[2:]
11061179
elif "." not in reference_match.group(1) and "[*]" not in reference_match.group(1):
11071180
match = reference_match.group(1).split("[")[0]
11081181
else:
@@ -1771,7 +1844,7 @@ def string_substitution(string, pattern, row, term, ignore, iterator):
17711844
if row == None:
17721845
return None
17731846
if match in row.keys():
1774-
if row[match] != None and row[match] != "nan" and row[match] != "N/A" and row[match] != "None":
1847+
if row[match] != None and row[match] != "nan" and row[match] != "N/A" and row[match] != "None" and row[match] != "NULL":
17751848
if (type(row[match]).__name__) != "str" and row[match] != None:
17761849
if (type(row[match]).__name__) == "float":
17771850
row[match] = repr(row[match])
@@ -1834,7 +1907,7 @@ def string_substitution(string, pattern, row, term, ignore, iterator):
18341907
print("The index needs to be indicated.\n")
18351908
return None
18361909
else:
1837-
if row[match] != None and row[match] != "nan" and row[match] != "N/A" and row[match] != "None":
1910+
if row[match] != None and row[match] != "nan" and row[match] != "N/A" and row[match] != "None" and row[match] != "NULL":
18381911
if re.search("^[\s|\t]*$", row[match]) is None:
18391912
new_string = new_string[:start] + row[match].strip().replace("\"", "'") + new_string[end:]
18401913
new_string = "\"" + new_string + "\"" if new_string[0] != "\"" and new_string[-1] != "\"" else new_string
@@ -2135,7 +2208,9 @@ def clean_URL_suffix(URL_suffix):
21352208
return cleaned_URL
21362209

21372210
def string_separetion(string):
2138-
if ("{" in string) and ("[" in string):
2211+
if "\\{" in string:
2212+
return "", ""
2213+
elif ("{" in string) and ("[" in string):
21392214
prefix = string.split("{")[0]
21402215
condition = string.split("{")[1].split("}")[0]
21412216
postfix = string.split("{")[1].split("}")[1]

‎rdfizer/rdfizer/lv_functions.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def hash_maker_lv_list(parent_data, parent_value, parent_id, parent, child):
107107
for row in parent_data:
108108
parent_values = ""
109109
for parent_attr in parent:
110-
if isinstance(parent,dict):
110+
if isinstance(parent_attr,dict):
111111
if parent_attr["type"] == "template":
112112
value = string_substitution_json(parent_attr["value"], "{(.+?)}", row, "subject", "yes", "")
113113
if value != None:
@@ -119,8 +119,14 @@ def hash_maker_lv_list(parent_data, parent_value, parent_id, parent, child):
119119
parent_values += "_" + row[parent_attr]
120120

121121
if parent_values[1:] in hash_table:
122-
if parent_value in row:
122+
if isinstance(parent_value,dict):
123+
value = parent_value["value"]
124+
elif parent_value in row:
123125
value = row[parent_value]
126+
elif "{" in parent_value:
127+
value = string_substitution_json(parent_value, "{(.+?)}", row, "subject", "yes", "")
128+
if value == None:
129+
value = ""
124130
else:
125131
value = ""
126132
if duplicate == "yes":
@@ -131,8 +137,14 @@ def hash_maker_lv_list(parent_data, parent_value, parent_id, parent, child):
131137

132138

133139
else:
134-
if parent_value in row:
140+
if isinstance(parent_value,dict):
141+
value = parent_value["value"]
142+
elif parent_value in row:
135143
value = row[parent_value]
144+
elif "{" in parent_value:
145+
value = string_substitution_json(parent_value, "{(.+?)}", row, "subject", "yes", "")
146+
if value == None:
147+
value = ""
136148
else:
137149
value = ""
138150
hash_table.update({parent_values[1:]: {value: "object"}})
@@ -489,6 +501,11 @@ def view_projection(view_map, view_map_list, internal_maps):
489501
new_row[attr["name"]+".#"] = 0
490502
else:
491503
new_row[attr["name"]] = list(lv_join_table[parent_view + "_" + attr["child_condition"]][row[attr["child_condition"]]].keys())
504+
else:
505+
if attr["type"] == "inner_join":
506+
inner_join_success = False
507+
else:
508+
new_row[attr["name"]] = None
492509
elif attr["child_condition"] in new_row:
493510
if new_row[attr["child_condition"]] in lv_join_table[parent_view + "_" + attr["child_condition"]]:
494511
iterable = True
@@ -497,6 +514,11 @@ def view_projection(view_map, view_map_list, internal_maps):
497514
new_row[attr["name"]+".#"] = 0
498515
else:
499516
new_row[attr["name"]] = list(lv_join_table[parent_view + "_" + attr["child_condition"]][new_row[attr["child_condition"]]].keys())
517+
else:
518+
if attr["type"] == "inner_join":
519+
inner_join_success = False
520+
else:
521+
new_row[attr["name"]] = None
500522
else:
501523
if attr["type"] == "inner_join":
502524
inner_join_success = False

0 commit comments

Comments
 (0)