diff --git a/scripts/pretokenize/README.md b/scripts/datasets/pretokenize/README.md similarity index 100% rename from scripts/pretokenize/README.md rename to scripts/datasets/pretokenize/README.md diff --git a/scripts/pretokenize/process_dataset.py b/scripts/datasets/pretokenize/process_dataset.py similarity index 100% rename from scripts/pretokenize/process_dataset.py rename to scripts/datasets/pretokenize/process_dataset.py diff --git a/scripts/pretokenize/sky.yaml b/scripts/datasets/pretokenize/sky.yaml similarity index 96% rename from scripts/pretokenize/sky.yaml rename to scripts/datasets/pretokenize/sky.yaml index 0236881a9..49958bcf8 100644 --- a/scripts/pretokenize/sky.yaml +++ b/scripts/datasets/pretokenize/sky.yaml @@ -51,7 +51,7 @@ run: | set -x mkdir -p "${OUTPUT_DATA}/${DATASET_SPLIT}" - python3 ./src/experimental/pretokenize/process_dataset.py \ + python3 ./scripts/datasets/pretokenize/process_dataset.py \ --verbose \ --input_dataset "${INPUT_DATASET}" \ --dataset_split "${DATASET_SPLIT}" \