Merge pull request #620 from Kanaries/feat-compress

feat: compress all data in html
Kanaries · Sep 19, 2024 · 6c72074 · 6c72074
2 parents 6711936 + 763dd12
commit 6c72074
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 84 deletions.
diff --git a/app/package.json b/app/package.json
@@ -37,7 +37,6 @@
     "lucide-react": "^0.341.0",
     "mobx": "^6.9.0",
     "mobx-react-lite": "^3.4.3",
-    "pako": "^2.1.0",
     "postcss": "^8.3.7",
     "react": "^18.x",
     "react-dom": "^18.x",

diff --git a/app/src/components/preview/index.tsx b/app/src/components/preview/index.tsx
@@ -1,7 +1,6 @@
 import React from "react";
 import { observer } from "mobx-react-lite";
-import pako from "pako";
-import { PureRenderer } from '@kanaries/graphic-walker';
+import { PureRenderer, IRow } from '@kanaries/graphic-walker';
 import type { IDarkMode, IThemeKey } from '@kanaries/graphic-walker/interfaces';
 
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -13,45 +12,12 @@ interface IPreviewProps {
     dark: IDarkMode;
     charts: {
         visSpec: any;
-        data: string;
+        data: IRow[];
     }[];
 }
 
-const getInflateData = (dataStr: string) => {
-    let binaryString = atob(dataStr);
-    let compressed = new Uint8Array(binaryString.length);
-    for (let i = 0; i < binaryString.length; i++) {
-        compressed[i] = binaryString.charCodeAt(i);
-    }
-    const inflated = pako.inflate(compressed, {to: "string"});
-
-    const datas = JSON.parse(inflated);
-    const keys = Object.keys(datas);
-    if (keys.length === 0) {
-        return [];
-    }
-    const count = datas[keys[0]].length;
-
-    const result = [] as any[];
-    for (let i = 0; i < count; i++) {
-        let item = {};
-        keys.forEach(key => {
-            item[key] = datas[key][i]
-        })
-        result.push(item);
-    }
-
-    return result;
-}
-
 const Preview: React.FC<IPreviewProps> = observer((props) => {
     const { charts, themeKey } = props;
-    const formatedCharts = charts.map((chart) => {
-        return {
-            ...chart,
-            data: getInflateData(chart.data)
-        }
-    })
 
     return (
         <React.StrictMode>
@@ -60,12 +26,12 @@ const Preview: React.FC<IPreviewProps> = observer((props) => {
                 <Tabs defaultValue="0" className="w-full">
                     <div className="overflow-x-auto max-w-full">
                         <TabsList>
-                            {formatedCharts.map((chart, index) => {
+                            {charts.map((chart, index) => {
                                 return <TabsTrigger key={index} value={index.toString()}>{chart.visSpec.name}</TabsTrigger>
                             })}
                         </TabsList>
                     </div>
-                    {formatedCharts.map((chart, index) => {
+                    {charts.map((chart, index) => {
                         return <TabsContent key={index} value={index.toString()}>
                             <PureRenderer
                                 vizThemeConfig={themeKey as IThemeKey}
@@ -89,13 +55,12 @@ interface IChartPreviewProps {
     themeKey: string;
     dark: IDarkMode;
     visSpec: any;
-    data: string;
+    data: IRow[];
     title: string;
     desc: string;
 }
 
 const ChartPreview: React.FC<IChartPreviewProps> = observer((props) => {
-    const formatedData = getInflateData(props.data);
 
     return (
         <React.StrictMode>
@@ -109,7 +74,7 @@ const ChartPreview: React.FC<IChartPreviewProps> = observer((props) => {
                     visualLayout={props.visSpec.layout}
                     visualState={props.visSpec.encodings}
                     type='remote'
-                    computation={async(_) => { return formatedData }}
+                    computation={async(_) => { return props.data }}
                     appearance={props.dark as IDarkMode}
                 />
             </div>

diff --git a/app/yarn.lock b/app/yarn.lock
@@ -5343,11 +5343,6 @@ pad-left@^2.1.0:
   dependencies:
     repeat-string "^1.5.4"
 
-pako@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
-  integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==
-
 papaparse@^5.1.1:
   version "5.4.1"
   resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.4.1.tgz#f45c0f871853578bd3a30f92d96fdcfb6ebea127"

diff --git a/pygwalker/services/preview_image.py b/pygwalker/services/preview_image.py
@@ -10,7 +10,7 @@
 from pygwalker.utils.encode import DataFrameEncoder
 from pygwalker.utils.display import display_html
 from pygwalker.utils.randoms import generate_hash_code
-from pygwalker.services.render import jinja_env, GWALKER_SCRIPT_BASE64
+from pygwalker.services.render import jinja_env, GWALKER_SCRIPT_BASE64, compress_data
 
 
 class ImgData(BaseModel):
@@ -31,20 +31,6 @@ class ChartData(BaseModel):
     title: str
 
 
-def _compress_data(data: List[List[Dict[str, Any]]]) -> str:
-    formated_data = {}
-    if data:
-        keys = list(data[0].keys())
-        formated_data = {key: [] for key in keys}
-        for item in data:
-            for key in keys:
-                formated_data[key].append(item[key])
-
-    data_json_str = json.dumps(formated_data, cls=DataFrameEncoder)
-    data_base64_str = base64.b64encode(zlib.compress(data_json_str.encode())).decode()
-    return data_base64_str
-
-
 def render_gw_preview_html(
     vis_spec_obj: List[Dict[str, Any]],
     datas: List[List[Dict[str, Any]]],
@@ -60,10 +46,9 @@ def render_gw_preview_html(
         vis_spec_obj,
         datas
     ):
-        data_base64_str = _compress_data(data)
         charts.append({
             "visSpec": vis_spec_item,
-            "data": data_base64_str
+            "data": data
         })
 
     props = {"charts": charts, "themeKey": theme_key, "dark": appearance, "gid": gid}
@@ -75,7 +60,7 @@ def render_gw_preview_html(
             'id': container_id,
             'gw_script': GWALKER_SCRIPT_BASE64,
             "component_script": "PyGWalkerApp.PreviewApp(props, gw_id);",
-            "props": json.dumps(props, cls=DataFrameEncoder)
+            "props": compress_data(json.dumps(props, cls=DataFrameEncoder))
         },
         component_url=""
     )
@@ -97,7 +82,7 @@ def render_gw_chart_preview_html(
 
     props = {
         "visSpec": single_vis_spec,
-        "data": _compress_data(data),
+        "data": data,
         "themeKey": theme_key,
         "title": title,
         "desc": desc,
@@ -111,7 +96,7 @@ def render_gw_chart_preview_html(
             'id': container_id,
             'gw_script': GWALKER_SCRIPT_BASE64,
             "component_script": "PyGWalkerApp.ChartPreviewApp(props, gw_id);",
-            "props": json.dumps(props, cls=DataFrameEncoder)
+            "props": compress_data(json.dumps(props, cls=DataFrameEncoder))
         },
         component_url=""
     )

diff --git a/pygwalker/services/render.py b/pygwalker/services/render.py
@@ -3,6 +3,7 @@
 import base64
 import html as m_html
 from typing import Dict, List, Any, Optional
+import zlib
 
 from jinja2 import Environment, PackageLoader
 
@@ -17,9 +18,17 @@
     autoescape=(()),  # select_autoescape()
 )
 
+
+def compress_data(data: str) -> str:
+    compress = zlib.compressobj(zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, 15, 8, 0)
+    compressed_data = compress.compress(data.encode())
+    compressed_data += compress.flush()
+    return base64.b64encode(compressed_data).decode()
+
+
 with open(os.path.join(ROOT_DIR, 'templates', 'dist', 'pygwalker-app.iife.js'), 'r', encoding='utf8') as f:
     GWALKER_SCRIPT = f.read()
-    GWALKER_SCRIPT_BASE64 = base64.b64encode(GWALKER_SCRIPT.encode()).decode()
+    GWALKER_SCRIPT_BASE64 = compress_data(GWALKER_SCRIPT)
 
 
 def get_max_limited_datas(datas: List[Dict[str, Any]], byte_limit: int) -> List[Dict[str, Any]]:
@@ -65,7 +74,7 @@ def render_gwalker_html(gid: int, props: Dict[str, Any]) -> str:
             'id': container_id,
             'gw_script': GWALKER_SCRIPT_BASE64,
             "component_script": "PyGWalkerApp.GWalker(props, gw_id);",
-            "props": json.dumps(props, cls=DataFrameEncoder)
+            "props": compress_data(json.dumps(props, cls=DataFrameEncoder)),
         },
         component_url=GlobalVarManager.component_url
     )

diff --git a/pygwalker/templates/pygwalker_main_page.html b/pygwalker/templates/pygwalker_main_page.html
@@ -10,30 +10,34 @@
         Loading Graphic-Walker UI...
     </div>
     <script>
-        function decodeGwScript(gw_script) {
-            var decodedBase64 = atob(gw_script);
-            var textDecoder = new TextDecoder('utf-8');
-            return textDecoder.decode(new Uint8Array(decodedBase64.split('').map(char => char.charCodeAt(0))));
-        }
-        eval(decodeGwScript("{{ gwalker.gw_script }}"));
-
+        var gw_script = "{{ gwalker.gw_script }}";
         var gw_id = "{{ gwalker.id }}";
-        var props = {{ gwalker.props }};
-        try{
-            window.__GW_VERSION=props.version;
-            {{ gwalker.component_script }};
-        } catch(e) {
-            console.error(e);
+        var props_data = "{{ gwalker.props }}";
+        async function runGwScript() {
+            const script_stream = await fetch('data:application/octet-stream;base64,' + gw_script).then((res) => res.body.pipeThrough(new DecompressionStream('deflate')));
+            const script = await new Response(script_stream).text();
+            eval(script);
+            const props_stream = await fetch('data:application/octet-stream;base64,' + props_data).then((res) => res.body.pipeThrough(new DecompressionStream('deflate')));
+            const props = await new Response(props_stream).json();
+            try{
+                window.__GW_VERSION=props.version;
+                {{ gwalker.component_script }};
+            } catch(e) {
+                console.error(e);
+            }
         }
+        runGwScript();
     </script>
 </body>
 {% endif %}
 
 {% if component_url != "" %}
 <body>
     <script>   
-        window.onload = function() {
-            const props = {{ gwalker.props }};
+        window.onload = async function() {
+            const props_data = "{{ gwalker.props }}";
+            const props_stream = await fetch('data:application/octet-stream;base64,' + props_data).then((res) => res.body.pipeThrough(new DecompressionStream('deflate')));
+            const props = await new Response(props_stream).json();
             const iframe = window.parent.document.getElementById("gwalker-" + props.id);   
             iframe.onload = function() {
                 iframe.contentWindow.postMessage({data: props, type: "pyg_props"}, "*");