Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

script to parallelize uploads (GDEV-1502) #3

Merged
merged 21 commits into from
Jan 18, 2023
Merged
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions src/otp_tsv_to_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def load_file_metadata(otp_tsv: Path) -> list[FileMetadata]:
with open(otp_tsv, "r", encoding="utf-8") as tsv_file:
files = [
FileMetadata(
path=Path(line.split("\t")[0].strip()),
path=Path(line.split("\t")[0].strip()).absolute(),
alias=line.split("\t")[1].strip(),
)
for line in tsv_file.readlines()
Expand All @@ -67,7 +67,7 @@ def load_file_metadata(otp_tsv: Path) -> list[FileMetadata]:


def check_file_upload(file: FileMetadata, output_dir: Path) -> bool:
"""Returns true if the file was already uploaded. Returns falls otherwise."""
"""Returns true if the file was already uploaded. Returns false otherwise."""

output_yaml = output_dir / f"{file.alias}.json"
return output_yaml.exists()
Expand Down Expand Up @@ -191,10 +191,7 @@ def main(
parallel_processes: int = typer.Option(..., help="Number of parallel uploads."),
dry_run: bool = typer.Option(
False,
help=(
"Only show the command lines that would be used."
+ " No uploads are performed."
),
help=("Only print commands for each file." + " No uploads are performed."),
),
):
"""
Expand Down