Writing binary files

CHANGELOG.md

@@ -6,9 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+### Added
+
+- Image read and write support (#1171)
+- `imageio` support (#1171)
+
 ### Fixed
 
 - Bugs in append mode for writing to files in python (#1200)
+- Writing binary files to the `pyodide` filesystem (#1171)
 
 ## [0.29.1] - 2025-02-21
 

cypress/e2e/spec-wc-pyodide.cy.js

@@ -140,7 +140,7 @@ describe("Running the code with pyodide", () => {
     cy.get("editor-wc")
       .shadow()
       .find(".cm-editor")
-      .should("contain", "Hello again world");
+      .should("contain", "Hello worldHello again world");
   });
 
   it("runs a simple program with a built-in python module", () => {

src/PyodideWorker.js

@@ -40,7 +40,12 @@ const PyodideWorker = () => {
 
     switch (data.method) {
       case "writeFile":
-        pyodide.FS.writeFile(data.filename, encoder.encode(data.content));
+        if (data.content instanceof ArrayBuffer) {
+          const dataArray = new Uint8Array(data.content);
+          pyodide.FS.writeFile(data.filename, dataArray);
+        } else {
+          pyodide.FS.writeFile(data.filename, encoder.encode(data.content));
+        }
         break;
       case "runPython":
         runPython(data.python);
@@ -97,25 +102,31 @@ const PyodideWorker = () => {
       import basthon
       import builtins
       import os
+      import mimetypes
 
       MAX_FILES = 100
       MAX_FILE_SIZE = 8500000
 
       def _custom_open(filename, mode="r", *args, **kwargs):
           if "x" in mode and os.path.exists(filename):
               raise FileExistsError(f"File '{filename}' already exists")
-          if ("w" in mode or "a" in mode or "x" in mode) and "b" not in mode:
+          if "w" in mode or "a" in mode or "x" in mode:
               if len(os.listdir()) > MAX_FILES and not os.path.exists(filename):
                   raise OSError(f"File system limit reached, no more than {MAX_FILES} files allowed")
               class CustomFile:
                   def __init__(self, filename):
                       self.filename = filename
-                      self.content = ""
+                      type = mimetypes.guess_type(filename)[0]
+                      if type and "text" in type:
+                          self.content = ""
+                      else:
+                          self.content = b''
 
                   def write(self, content):
                       self.content += content
                       if len(self.content) > MAX_FILE_SIZE:
                           raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes")
+
                       with _original_open(self.filename, mode) as f:
                           f.write(self.content)
                       basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode })
@@ -367,6 +378,33 @@ const PyodideWorker = () => {
         `);
       },
     },
+    imageio: {
+      before: async () => {
+        await pyodide.loadPackage("imageio");
+        await pyodide.loadPackage("requests");
+        pyodide.runPython(`
+          import imageio.v3 as iio
+          import io
+          import requests
+
+          # Store the original imread function to avoid recursion
+          #_original_imread = iio.imread  
+
+          def custom_imread(uri, *args, **kwargs):
+              split_uri = uri.split(":")
+              if split_uri[0] == "imageio":
+                  new_url = f"https://raw.githubusercontent.com/imageio/imageio-binaries/master/images/{split_uri[1]}"
+                  response = requests.get(new_url, stream=True)
+                  response.raise_for_status()
+                  return _original_imread(io.BytesIO(response.content), *args, **kwargs)  # Use the original imread
+              return _original_imread(uri, *args, **kwargs)  # Call the original function for all other cases
+
+          # Override iio.imread
+          iio.imread = custom_imread
+          `);
+      },
+      after: () => {},
+    },
   };
 
   const fakeBasthonPackage = {
@@ -427,6 +465,12 @@ const PyodideWorker = () => {
     _original_open = builtins.open
     `);
 
+    await pyodide.loadPackage("imageio");
+    await pyodide.runPythonAsync(`
+    import imageio.v3 as iio
+    _original_imread = iio.imread
+    `);
+
     await pyodide.loadPackage("pyodide-http");
     await pyodide.runPythonAsync(`
         import pyodide_http
@@ -473,6 +517,8 @@ const PyodideWorker = () => {
   const parsePythonError = (error) => {
     const type = error.type;
     const [trace, info] = error.message.split(`${type}:`).map((s) => s?.trim());
+    console.log(trace);
+    console.log(info);
 
     const lines = trace.split("\n");
 

src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx

@@ -1,6 +1,12 @@
 /* eslint-disable react-hooks/exhaustive-deps */
 import "../../../../../assets/stylesheets/PythonRunner.scss";
-import React, { useContext, useEffect, useRef, useState } from "react";
+import React, {
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
 import { useDispatch, useSelector } from "react-redux";
 import { useTranslation } from "react-i18next";
 import classNames from "classnames";
@@ -10,6 +16,7 @@ import {
   setLoadedRunner,
   updateProjectComponent,
   addProjectComponent,
+  updateImages,
 } from "../../../../../redux/EditorSlice";
 import { Tab, Tabs, TabList, TabPanel } from "react-tabs";
 import { useMediaQuery } from "react-responsive";
@@ -20,6 +27,7 @@ import VisualOutputPane from "./VisualOutputPane";
 import OutputViewToggle from "../OutputViewToggle";
 import { SettingsContext } from "../../../../../utils/settings";
 import RunnerControls from "../../../../RunButton/RunnerControls";
+import store from "../../../../../redux/stores/WebComponentStore";
 
 const getWorkerURL = (url) => {
   const content = `
@@ -51,6 +59,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
   const stdinClosed = useRef();
   const loadedRunner = useSelector((state) => state.editor.loadedRunner);
   const projectImages = useSelector((s) => s.editor.project.image_list);
+  const projectImageNames = projectImages?.map((image) => image.filename);
   const projectCode = useSelector((s) => s.editor.project.components);
   const projectIdentifier = useSelector((s) => s.editor.project.identifier);
   const focussedFileIndex = useSelector(
@@ -124,7 +133,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
         }
       };
     }
-  }, [pyodideWorker, projectCode, openFiles, focussedFileIndex]);
+  }, [pyodideWorker, projectCode, projectImages, openFiles, focussedFileIndex]);
 
   useEffect(() => {
     if (codeRunTriggered && active && output.current) {
@@ -213,11 +222,81 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
     disableInput();
   };
 
-  const handleFileWrite = (filename, content, mode, cascadeUpdate) => {
+  const fileWriteQueue = useRef([]); // Queue to store file write requests
+  const isExecuting = useRef(false);
+
+  const handleFileWrite = useCallback(
+    async (filename, content, mode, cascadeUpdate) => {
+      // Add the file write request to the queue
+      console.log(`Writing ${content} to ${filename}`);
+      fileWriteQueue.current.push({
+        filename,
+        content,
+        mode,
+        cascadeUpdate,
+        projectImages,
+      });
+
+      // Process the queue if not already executing
+      if (!isExecuting.current) {
+        processFileWriteQueue();
+      }
+    },
+    [projectImages, projectImageNames],
+  );
+
+  const processFileWriteQueue = useCallback(async () => {
+    if (fileWriteQueue.current.length === 0) {
+      isExecuting.current = false;
+      return;
+    }
+
+    isExecuting.current = true;
+    const { filename, content, mode, cascadeUpdate } =
+      fileWriteQueue.current.shift();
+
     const [name, extension] = filename.split(".");
     const componentToUpdate = projectCode.find(
       (item) => item.extension === extension && item.name === name,
     );
+
+    if (mode === "wb" || mode === "w+b") {
+      const { uploadImages, updateImage } = ApiCallHandler({
+        reactAppApiEndpoint,
+      });
+
+      console.log("the state of the store is: ");
+      console.log(store.getState());
+      const projectImageNames = (
+        store.getState().editor.project.image_list || []
+      ).map((image) => image.filename);
+      console.log("Project Image Names: ", projectImageNames);
+      console.log(filename);
+      console.log(filename.split("/").pop());
+      if (projectImageNames.includes(filename.split("/").pop())) {
+        console.log("Image already exists");
+        const response = await updateImage(
+          projectIdentifier,
+          user.access_token,
+          // file object with the correct filename and binary content
+          new File([content], filename, { type: "application/octet-stream" }),
+        );
+        if (response.status === 200) {
+          dispatch(updateImages(response.data.image_list));
+        }
+        processFileWriteQueue(projectImageNames); // Process the next item in the queue
+        return;
+      }
+      const response = await uploadImages(
+        projectIdentifier,
+        user.access_token,
+        // file object with the correct filename and binary content
+        [new File([content], filename, { type: "application/octet-stream" })],
+      );
+      dispatch(updateImages(response.data.image_list));
+      processFileWriteQueue(projectImageNames); // Process the next item in the queue
+      return;
+    }
     let updatedContent;
     if (mode === "w" || mode === "x") {
       updatedContent = content;
@@ -240,7 +319,9 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => {
         addProjectComponent({ name, extension, content: updatedContent }),
       );
     }
-  };
+
+    processFileWriteQueue(); // Process the next item in the queue
+  }, [projectImages, projectImageNames]);
 
   const handleVisual = (origin, content) => {
     if (showVisualOutputPanel) {

src/utils/apiCallHandler.js

@@ -106,6 +106,17 @@ const ApiCallHandler = ({ reactAppApiEndpoint }) => {
     );
   };
 
+  const updateImage = async (projectIdentifier, accessToken, image) => {
+    var formData = new FormData();
+    formData.append("image", image, image.name);
+
+    return await put(
+      `${host}/api/projects/${projectIdentifier}/images`,
+      formData,
+      { ...headers(accessToken), "Content-Type": "multipart/form-data" },
+    );
+  };
+
   const createError = async (
     projectIdentifier,
     userId,
@@ -137,6 +148,7 @@ const ApiCallHandler = ({ reactAppApiEndpoint }) => {
     loadAssets,
     readProjectList,
     uploadImages,
+    updateImage,
     createError,
   };
 };