Merge pull request #105 from JetBrains-Research/fix-astminer

Fix astminer
JetBrains-Research · May 29, 2021 · fcc3751 · fcc3751
2 parents 790e8f7 + caf3fc3
commit fcc3751
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/code2seq/preprocessing/astminer_to_code2seq.py b/code2seq/preprocessing/astminer_to_code2seq.py
@@ -10,7 +10,10 @@
 
 
 def _get_id2value_from_csv(path_: str) -> Dict[str, str]:
-    return dict(numpy.genfromtxt(path_, delimiter=",", dtype=(str, str))[1:])
+    with open(path_, "r") as f:
+        lines = f.read().strip().split("\n")[1:]
+        parsed_lines = [line.split(",", maxsplit=1) for line in lines]
+        return {k: v for k, v in parsed_lines}
 
 
 def preprocess_csv(data_folder: str, dataset_name: str, holdout_name: str, is_shuffled: bool):

diff --git a/scripts/download_data.sh b/scripts/download_data.sh
@@ -9,7 +9,7 @@ LOAD_SPLITTED=false
 DATA_DIR=./data
 POJ_DOWNLOAD_SCRIPT=./scripts/download_poj.sh
 CODEFORCES_DOWNLOAD_SCRIPT=./scripts/download_codeforces.sh
-ASTMINER_PATH=../astminer/build/shadow/lib-0.*.jar
+ASTMINER_PATH=../astminer/build/shadow/astminer.jar
 SPLIT_SCRIPT=./scripts/split_dataset.sh
 
 function is_int(){