Skip to content

Commit

Permalink
update: pass correct compute
Browse files Browse the repository at this point in the history
  • Loading branch information
@VsevolodX
VsevolodX committed Feb 17, 2025
1 parent 051554a commit 3465eed
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 50 deletions.
2 changes: 1 addition & 1 deletion examples/job/get-file-from-job.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@
"source": [
"from utils.jupyterlite import download_content_to_file\n",
"\n",
"download_content_to_file(output_file_metadata[\"name\"], output_file)"
"download_content_to_file(content=output_file, filename=output_file_metadata[\"name\"])"
],
"outputs": [],
"execution_count": null
Expand Down
6 changes: 3 additions & 3 deletions examples/job/ml-train-model-predict-properties.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,11 +236,11 @@
"id": "wiudje4-2Qxc"
},
"source": [
"PPN = \"1\"\n",
"PPN = 1\n",
"QUEUE = \"D\"\n",
"NODES = \"1\"\n",
"NODES = 1\n",
"TIME_LIMIT = \"01:00:00\"\n",
"CLUSTER = \"cluster-001\""
"CLUSTER = apiConfig[\"clusters\"][0][\"displayName\"]"
],
"outputs": [],
"execution_count": null
Expand Down
135 changes: 89 additions & 46 deletions examples/workflow/qe_scf_calculation.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,8 @@
},
{
"cell_type": "code",
"execution_count": null,
"id": "62f59cb7",
"metadata": {},
"outputs": [],
"source": [
"# @title Authorization Form\n",
"ACCOUNT_ID = \"ACCOUNT_ID\" # @param {type:\"string\"}\n",
Expand Down Expand Up @@ -82,35 +80,79 @@
" from utils.jupyterlite import install_packages\n",
"\n",
" await install_packages(\"api_examples\")"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "504fd5dd-5411-4073-be50-62dd84baeb9b",
"metadata": {},
"cell_type": "markdown",
"source": [
"#### Compute\n",
"\n",
"Setup compute parameters. See [this](https://docs.mat3ra.com/infrastructure/compute-settings/ui) for more information about compute parameters.\n",
"\n",
"- **NODES**: Number of nodes. Defaults to 1.\n",
"- **PPN**: Number of MPI processes per each node, Defaults to 1.\n",
"- **QUEUE**: The name of queue to submit the jobs into. Defaults to D.\n",
"- **TIME_LIMIT**: Job walltime. Defaults to \"01:00:00\" (one hour).\n",
"- **CLUSTER**: The full qualified domain name (FQDN) or alias of the cluster to submit the jobs into."
],
"id": "5dca0feaf2c1b284"
},
{
"metadata": {
"id": "wiudje4-2Qxc"
},
"cell_type": "code",
"source": [
"PPN = 1\n",
"QUEUE = \"D\"\n",
"NODES = 1\n",
"TIME_LIMIT = \"01:00:00\"\n",
"CLUSTER = apiConfig[\"clusters\"][0][\"displayName\"]"
],
"id": "6b645d1149740c3f",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
"source": "### Initialize the endpoints",
"id": "a2fe3a7420ec31c3"
},
{
"metadata": {
"jupyter": {
"is_executing": true
}
},
"cell_type": "code",
"source": [
"from utils.settings import ENDPOINT_ARGS, ACCOUNT_ID\n",
"from utils.generic import wait_for_jobs_to_finish\n",
"\n",
"from exabyte_api_client.endpoints.workflows import WorkflowEndpoints\n",
"from exabyte_api_client.endpoints.materials import MaterialEndpoints\n",
"from exabyte_api_client.endpoints.jobs import JobEndpoints"
]
],
"id": "504fd5dd-5411-4073-be50-62dd84baeb9b",
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "b4e2b1f6-40be-4a7e-aa49-1fa36df4e33c",
"metadata": {},
"outputs": [],
"source": [
"# Initialize a helper class to interact with WorkflowEndpoints\n",
"workflow_endpoints = WorkflowEndpoints(*ENDPOINT_ARGS)\n",
"material_endpoints = MaterialEndpoints(*ENDPOINT_ARGS)\n",
"job_endpoints = JobEndpoints(*ENDPOINT_ARGS)"
]
],
"outputs": [],
"execution_count": null
},
{
"attachments": {},
Expand All @@ -132,10 +174,8 @@
},
{
"cell_type": "code",
"execution_count": null,
"id": "67bdb4e0-f7b4-4097-b5b9-66c8debaedd3",
"metadata": {},
"outputs": [],
"source": [
"# user modifiable part\n",
"script = r\"\"\"#!/bin/bash\n",
Expand Down Expand Up @@ -174,25 +214,25 @@
" Si 0.25 0.25 0.25\n",
"\n",
"K_POINTS (automatic)\n",
" 7 7 7 0 0 0\n",
" 3 3 3 0 0 0\n",
"EOF\n",
"# -------------------------- PSEUDO POTENTIAL FILE --------------------------- #\n",
"# provide a downloadable link for the pseudo potential file\n",
"wget https://media.githubusercontent.com/media/exabyte-io/api-examples/dev/examples/assets/Si.pz-vbc.UPF\n",
"# --------------------------------- RUN JOB ---------------------------------- #\n",
"# load required module\n",
"module add espresso/63-i-174-impi-044\n",
"module add espresso/6.3-g-4.8.5-ompi-1.10.0\n",
"\n",
"mpirun -np $PBS_NP pw.x -in pw.in | tee pw.out\n",
"\"\"\""
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "8edba851-6207-40ba-beff-ce2b59a1cdfa",
"metadata": {},
"outputs": [],
"source": [
"# populate workflow from a template\n",
"import json\n",
Expand All @@ -201,19 +241,21 @@
" WORKFLOW_BODY = json.load(f)\n",
"\n",
"WORKFLOW_BODY[\"subworkflows\"][0][\"units\"][0][\"input\"][0][\"content\"] = script"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "19037ef9-d5f7-4344-90ea-e7cbf8bbeba0",
"metadata": {},
"outputs": [],
"source": [
"# create workflow\n",
"OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or ACCOUNT_ID\n",
"WORKFLOW_RESP = workflow_endpoints.create(WORKFLOW_BODY, owner_id=OWNER_ID)"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "markdown",
Expand All @@ -227,66 +269,67 @@
},
{
"cell_type": "code",
"execution_count": null,
"id": "41d85194-5a2b-4e65-9302-251932d04c11",
"metadata": {},
"outputs": [],
"source": [
"compute = job_endpoints.get_compute(CLUSTER, PPN, NODES, QUEUE, TIME_LIMIT)\n",
"# job creation payload\n",
"JOB_BODY = {\n",
" \"owner\": {\"_id\": OWNER_ID},\n",
" \"name\": \"SCF Calculation\",\n",
" \"compute\": {\"ppn\": 4, \"nodes\": 1, \"queue\": \"OR\"},\n",
" \"compute\": compute,\n",
" \"workflow\": {\"_id\": WORKFLOW_RESP[\"_id\"]},\n",
"}"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ed28b99-923e-4b30-a0d5-51fe45675c77",
"metadata": {},
"outputs": [],
"source": [
"# create job\n",
"JOB_RESP = job_endpoints.create(JOB_BODY)"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "7dad9ab0-29e7-45a1-832d-364374371c64",
"metadata": {},
"outputs": [],
"source": [
"# TODO: NOT SUPPORTED YET, upload pseudo potential file\n",
"# with open('../assets/Si.pz-vbc.UPF', 'r') as f:\n",
"# PP_FILE = f.read()\n",
"# data = json.dumps({\"files\": PP_FILE})\n",
"# FILE_RESP=job_endpoints.insert_output_files(JOB_RESP[\"_id\"], data)"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "aa65092c-a30a-402e-85c2-506746e4664c",
"metadata": {},
"outputs": [],
"source": [
"# submit job to run\n",
"job_endpoints.submit(JOB_RESP[\"_id\"])"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "5129a629-a46a-4a4c-8de9-98228611aa87",
"metadata": {},
"outputs": [],
"source": [
"# monitor job and wait for it to be finished\n",
"wait_for_jobs_to_finish(job_endpoints, [JOB_RESP[\"_id\"]], 30)"
]
],
"outputs": [],
"execution_count": null
},
{
"attachments": {},
Expand All @@ -299,10 +342,8 @@
},
{
"cell_type": "code",
"execution_count": null,
"id": "87b8fa4b-39cb-4a11-b8da-b07303b9d42c",
"metadata": {},
"outputs": [],
"source": [
"files = job_endpoints.list_files(JOB_RESP[\"_id\"])\n",
"for file in files:\n",
Expand All @@ -312,28 +353,28 @@
"from mat3ra.utils.jupyterlite.url import read_from_url\n",
"\n",
"output_file = await read_from_url(output_file_metadata[\"signedUrl\"])"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "c160fc04-48c2-40da-b21a-db6706b93dd7",
"metadata": {},
"outputs": [],
"source": [
"energy = []\n",
"len_energy = len(\"total energy\")\n",
"for line in output_file.split(\"\\n\"):\n",
" if line.strip().lstrip(\"!\")[:len_energy] == \"total energy\":\n",
" energy.append(float(line.split(\"=\")[1].rstrip(\"Ry\")))"
]
],
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"execution_count": null,
"id": "e89a2984-10d7-4e80-929a-4f818a603d6b",
"metadata": {},
"outputs": [],
"source": [
"# plot energy with iteration step\n",
"import matplotlib.pyplot as plt\n",
Expand All @@ -344,7 +385,9 @@
"plt.xlabel(\"Number of iteration\")\n",
"plt.ylabel(\"Energy (Ry)\")\n",
"plt.show()"
]
],
"outputs": [],
"execution_count": null
}
],
"metadata": {
Expand Down

0 comments on commit 3465eed

Please # to comment.