We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent b376029 commit 0eb854cCopy full SHA for 0eb854c
1 file changed
benchmarks/P3L_mling.py
@@ -91,19 +91,19 @@ def get_wikitext2_text(tokenizer):
91
return test_enc, test_text
92
93
94
-def get_flores_plus_text(tokenizer, lng_scrpt):
+def get_flores_plus_text(tokenizer, lng_script):
95
hf_hub_download(
96
repo_id="alexei-v-ivanov-amd/flores_plus",
97
repo_type="dataset",
98
- filename=lng_scrpt + ".parquet",
+ filename=lng_script + ".parquet",
99
local_dir="./",
100
)
101
102
- df = pandas.read_parquet("./" + lng_scrpt + ".parquet")
+ df = pandas.read_parquet("./" + lng_script + ".parquet")
103
test_text = "\n\n".join(line.strip() for line in df["text"])
104
test_enc = tokenizer(test_text)
105
106
- os.remove("./" + lng_scrpt + ".parquet")
+ os.remove("./" + lng_script + ".parquet")
107
108
109
0 commit comments