merge main

oumi-ai · Feb 8, 2025 · 5060fb3 · 5060fb3
2 parents bec5176 + af50ccd
commit 5060fb3
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 1 deletion.
diff --git a/scripts/pretokenize/README.md → scripts/datasets/pretokenize/README.md b/scripts/pretokenize/README.md → scripts/datasets/pretokenize/README.md
diff --git a/scripts/pretokenize/process_dataset.py → ...s/datasets/pretokenize/process_dataset.py b/scripts/pretokenize/process_dataset.py → ...s/datasets/pretokenize/process_dataset.py
diff --git a/scripts/pretokenize/sky.yaml → scripts/datasets/pretokenize/sky.yaml b/scripts/pretokenize/sky.yaml → scripts/datasets/pretokenize/sky.yaml
@@ -51,7 +51,7 @@ run: |
 
   set -x
   mkdir -p "${OUTPUT_DATA}/${DATASET_SPLIT}"
-  python3 ./src/experimental/pretokenize/process_dataset.py \
+  python3 ./scripts/datasets/pretokenize/process_dataset.py \
     --verbose \
     --input_dataset "${INPUT_DATASET}" \
     --dataset_split "${DATASET_SPLIT}" \