From 763dd1205dffeb686e47a65feb4fc5017ae6944e Mon Sep 17 00:00:00 2001 From: islxyqwe Date: Thu, 19 Sep 2024 18:29:08 +0800 Subject: [PATCH] feat: compress all data in html --- app/package.json | 1 - app/src/components/preview/index.tsx | 47 +++----------------- app/yarn.lock | 5 --- pygwalker/services/preview_image.py | 25 +++-------- pygwalker/services/render.py | 13 +++++- pygwalker/templates/pygwalker_main_page.html | 34 +++++++------- 6 files changed, 41 insertions(+), 84 deletions(-) diff --git a/app/package.json b/app/package.json index aa8c6f4c..48efa730 100644 --- a/app/package.json +++ b/app/package.json @@ -37,7 +37,6 @@ "lucide-react": "^0.341.0", "mobx": "^6.9.0", "mobx-react-lite": "^3.4.3", - "pako": "^2.1.0", "postcss": "^8.3.7", "react": "^18.x", "react-dom": "^18.x", diff --git a/app/src/components/preview/index.tsx b/app/src/components/preview/index.tsx index 14833ba3..c9bf50b3 100644 --- a/app/src/components/preview/index.tsx +++ b/app/src/components/preview/index.tsx @@ -1,7 +1,6 @@ import React from "react"; import { observer } from "mobx-react-lite"; -import pako from "pako"; -import { PureRenderer } from '@kanaries/graphic-walker'; +import { PureRenderer, IRow } from '@kanaries/graphic-walker'; import type { IDarkMode, IThemeKey } from '@kanaries/graphic-walker/interfaces'; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; @@ -13,45 +12,12 @@ interface IPreviewProps { dark: IDarkMode; charts: { visSpec: any; - data: string; + data: IRow[]; }[]; } -const getInflateData = (dataStr: string) => { - let binaryString = atob(dataStr); - let compressed = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - compressed[i] = binaryString.charCodeAt(i); - } - const inflated = pako.inflate(compressed, {to: "string"}); - - const datas = JSON.parse(inflated); - const keys = Object.keys(datas); - if (keys.length === 0) { - return []; - } - const count = datas[keys[0]].length; - - const result = [] as any[]; - for (let i = 0; i < count; i++) { - let item = {}; - keys.forEach(key => { - item[key] = datas[key][i] - }) - result.push(item); - } - - return result; -} - const Preview: React.FC = observer((props) => { const { charts, themeKey } = props; - const formatedCharts = charts.map((chart) => { - return { - ...chart, - data: getInflateData(chart.data) - } - }) return ( @@ -60,12 +26,12 @@ const Preview: React.FC = observer((props) => {
- {formatedCharts.map((chart, index) => { + {charts.map((chart, index) => { return {chart.visSpec.name} })}
- {formatedCharts.map((chart, index) => { + {charts.map((chart, index) => { return = observer((props) => { - const formatedData = getInflateData(props.data); return ( @@ -109,7 +74,7 @@ const ChartPreview: React.FC = observer((props) => { visualLayout={props.visSpec.layout} visualState={props.visSpec.encodings} type='remote' - computation={async(_) => { return formatedData }} + computation={async(_) => { return props.data }} appearance={props.dark as IDarkMode} /> diff --git a/app/yarn.lock b/app/yarn.lock index d56dbb90..be673399 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -5343,11 +5343,6 @@ pad-left@^2.1.0: dependencies: repeat-string "^1.5.4" -pako@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86" - integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug== - papaparse@^5.1.1: version "5.4.1" resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.4.1.tgz#f45c0f871853578bd3a30f92d96fdcfb6ebea127" diff --git a/pygwalker/services/preview_image.py b/pygwalker/services/preview_image.py index 30d8ffd7..1c8ad352 100644 --- a/pygwalker/services/preview_image.py +++ b/pygwalker/services/preview_image.py @@ -10,7 +10,7 @@ from pygwalker.utils.encode import DataFrameEncoder from pygwalker.utils.display import display_html from pygwalker.utils.randoms import generate_hash_code -from pygwalker.services.render import jinja_env, GWALKER_SCRIPT_BASE64 +from pygwalker.services.render import jinja_env, GWALKER_SCRIPT_BASE64, compress_data class ImgData(BaseModel): @@ -31,20 +31,6 @@ class ChartData(BaseModel): title: str -def _compress_data(data: List[List[Dict[str, Any]]]) -> str: - formated_data = {} - if data: - keys = list(data[0].keys()) - formated_data = {key: [] for key in keys} - for item in data: - for key in keys: - formated_data[key].append(item[key]) - - data_json_str = json.dumps(formated_data, cls=DataFrameEncoder) - data_base64_str = base64.b64encode(zlib.compress(data_json_str.encode())).decode() - return data_base64_str - - def render_gw_preview_html( vis_spec_obj: List[Dict[str, Any]], datas: List[List[Dict[str, Any]]], @@ -60,10 +46,9 @@ def render_gw_preview_html( vis_spec_obj, datas ): - data_base64_str = _compress_data(data) charts.append({ "visSpec": vis_spec_item, - "data": data_base64_str + "data": data }) props = {"charts": charts, "themeKey": theme_key, "dark": appearance, "gid": gid} @@ -75,7 +60,7 @@ def render_gw_preview_html( 'id': container_id, 'gw_script': GWALKER_SCRIPT_BASE64, "component_script": "PyGWalkerApp.PreviewApp(props, gw_id);", - "props": json.dumps(props, cls=DataFrameEncoder) + "props": compress_data(json.dumps(props, cls=DataFrameEncoder)) }, component_url="" ) @@ -97,7 +82,7 @@ def render_gw_chart_preview_html( props = { "visSpec": single_vis_spec, - "data": _compress_data(data), + "data": data, "themeKey": theme_key, "title": title, "desc": desc, @@ -111,7 +96,7 @@ def render_gw_chart_preview_html( 'id': container_id, 'gw_script': GWALKER_SCRIPT_BASE64, "component_script": "PyGWalkerApp.ChartPreviewApp(props, gw_id);", - "props": json.dumps(props, cls=DataFrameEncoder) + "props": compress_data(json.dumps(props, cls=DataFrameEncoder)) }, component_url="" ) diff --git a/pygwalker/services/render.py b/pygwalker/services/render.py index 5db9f878..6a3ec3ae 100644 --- a/pygwalker/services/render.py +++ b/pygwalker/services/render.py @@ -3,6 +3,7 @@ import base64 import html as m_html from typing import Dict, List, Any, Optional +import zlib from jinja2 import Environment, PackageLoader @@ -17,9 +18,17 @@ autoescape=(()), # select_autoescape() ) + +def compress_data(data: str) -> str: + compress = zlib.compressobj(zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, 15, 8, 0) + compressed_data = compress.compress(data.encode()) + compressed_data += compress.flush() + return base64.b64encode(compressed_data).decode() + + with open(os.path.join(ROOT_DIR, 'templates', 'dist', 'pygwalker-app.iife.js'), 'r', encoding='utf8') as f: GWALKER_SCRIPT = f.read() - GWALKER_SCRIPT_BASE64 = base64.b64encode(GWALKER_SCRIPT.encode()).decode() + GWALKER_SCRIPT_BASE64 = compress_data(GWALKER_SCRIPT) def get_max_limited_datas(datas: List[Dict[str, Any]], byte_limit: int) -> List[Dict[str, Any]]: @@ -65,7 +74,7 @@ def render_gwalker_html(gid: int, props: Dict[str, Any]) -> str: 'id': container_id, 'gw_script': GWALKER_SCRIPT_BASE64, "component_script": "PyGWalkerApp.GWalker(props, gw_id);", - "props": json.dumps(props, cls=DataFrameEncoder) + "props": compress_data(json.dumps(props, cls=DataFrameEncoder)), }, component_url=GlobalVarManager.component_url ) diff --git a/pygwalker/templates/pygwalker_main_page.html b/pygwalker/templates/pygwalker_main_page.html index 64cc9065..e8edafc3 100644 --- a/pygwalker/templates/pygwalker_main_page.html +++ b/pygwalker/templates/pygwalker_main_page.html @@ -10,21 +10,23 @@ Loading Graphic-Walker UI... {% endif %} @@ -32,8 +34,10 @@ {% if component_url != "" %}