host selected packages (hbchannel & updater) on site

Author: mariotaku

pelicanconf.py

@@ -84,6 +84,12 @@
     ('utility', 'Utilities'),
 ]
 
+# Following packages will have their IPKs downloaded and hosted on the site
+HOST_PACKAGES: set[str] = {
+    'org.webosbrew.hbchannel',
+    'org.webosbrew.safeupdate'
+}
+
 DEFAULT_PAGINATION = 20
 
 # Uncomment following line if you want document-relative URLs when developing

repogen/apidata.py

@@ -4,9 +4,10 @@
 import os
 import urllib.parse
 from pathlib import Path
-from typing import List
+from typing import List, Set
 
 import more_itertools
+import requests
 from markdown import Markdown
 
 from repogen import pkg_info
@@ -31,10 +32,27 @@ def fix_manifest_url(item: PackageInfo, app_dir: Path):
     item['manifestUrl'] = manifest_url[manifest_url.find('/api/apps'):]
 
 
-def generate(packages: List[PackageInfo], outdir: Path):
+def save_ipk(item: PackageInfo, apps_dir: Path, site_url: str):
+    """
+    Save the ipk file to the apps directory, and modify PackageInfo to point to the local file.
+    """
+    manifest = item['manifest']
+    app_ipk = apps_dir.joinpath(item["id"], 'releases', f'{manifest["ipkHash"]["sha256"]}.ipk')
+    if not app_ipk.parent.exists():
+        app_ipk.parent.mkdir(parents=True, exist_ok=True)
+    # Not likely to happen, but just in case someday we host some ipk files by ourselves directly.
+    if site_url and manifest['ipkUrl'].startswith(site_url):
+        return
+    with requests.get(manifest['ipkUrl'], allow_redirects=True) as resp:
+        with ensure_open(app_ipk, 'wb') as f:
+            f.write(resp.content)
+    manifest['ipkUrl'] = f'{site_url.removesuffix("/")}/{"/".join(app_ipk.parts[-4:])}'
+
+
+def generate(packages: List[PackageInfo], api_dir: Path, apps_dir: Path = None, host_packages: Set[str] = None):
     markdown = Markdown()
 
-    appsdir: Path = outdir.joinpath('apps')
+    appsdir: Path = api_dir.joinpath('apps')
     site_url = siteurl()
 
     def package_item(p_info: PackageInfo, in_apps_dir: bool, is_details: bool) -> PackageInfo:
@@ -56,7 +74,7 @@ def package_item(p_info: PackageInfo, in_apps_dir: bool, is_details: bool) -> Pa
     max_page = math.ceil(packages_length / ITEMS_PER_PAGE)
 
     def save_page(page: int, items: [PackageInfo]):
-        json_file = appsdir.joinpath('%d.json' % page) if page > 1 else outdir.joinpath('apps.json')
+        json_file = appsdir.joinpath('%d.json' % page) if page > 1 else api_dir.joinpath('apps.json')
         with ensure_open(json_file, 'w', encoding='utf-8') as pf:
             json.dump({
                 'paging': {
@@ -71,13 +89,16 @@ def save_page(page: int, items: [PackageInfo]):
     chunks = more_itertools.chunked(packages, ITEMS_PER_PAGE) if packages else [[]]
     for index, chunk in enumerate(chunks):
         for item in chunk:
-            app_dir = appsdir.joinpath(item['id'])
-            releases_dir = app_dir.joinpath('releases')
-            fix_manifest_url(item, app_dir)
+            api_app_dir = appsdir.joinpath(item['id'])
+            releases_dir = api_app_dir.joinpath('releases')
+            if host_packages and item['id'] in host_packages:
+                save_ipk(item, apps_dir, site_url)
+            fix_manifest_url(item, api_app_dir)
+            # This will be used by dev-manager-desktop
             app_info = releases_dir.joinpath('latest.json')
             with ensure_open(app_info, 'w', encoding='utf-8') as f:
                 json.dump(package_item(item, True, True), f)
-            desc_html = app_dir.joinpath('full_description.html')
+            desc_html = api_app_dir.joinpath('full_description.html')
             with ensure_open(desc_html, 'w', encoding='utf-8') as f:
                 f.write(markdown.convert(item['description']))
         save_page(index + 1, chunk)

repogen/downloadipk.py

@@ -20,7 +20,7 @@
     except (requests.exceptions.JSONDecodeError, json.decoder.JSONDecodeError) as e:
         print(f'Could not parse manifest: {e}')
         exit(2)
-    except (IOError) as e:
+    except IOError as e:
         print(f'Could not open package info file: {e.strerror}')
         exit(3)
     except ValidationError as e:

repogen/plugin.py

@@ -30,7 +30,7 @@ def read(self, filename: str):
         info['manifest']['iconUri'] = info['iconUri']
         metadata = {
             'title': info['title'],
-            'override_save_as': f'apps/{info["id"]}.html',
+            'override_save_as': f'apps/{info["id"]}/index.html',
             'template': 'app',
             'status': 'hidden',
             'modified': info['lastmodified'],
@@ -93,12 +93,14 @@ def apps_list_href(page):
 
 def add_app_api_data(generator: StaticGenerator):
     packages = generator.settings['PACKAGES'].values()
+    output_path = generator.settings['OUTPUT_PATH']
+    host_packages = generator.settings.get('HOST_PACKAGES', None)
 
     def pool_list(pool: str):
         return list(sorted(filter(lambda pkg: pkg['pool'] == pool, packages), key=lambda pkg: pkg['title'].lower()))
 
-    apidata.generate(pool_list('main'), Path(generator.settings['OUTPUT_PATH'], 'api'))
-    apidata.generate(pool_list('non-free'), Path(generator.settings['OUTPUT_PATH'], 'api', 'non-free'))
+    apidata.generate(pool_list('main'), Path(output_path, 'api'), Path(output_path, 'apps'), host_packages)
+    apidata.generate(pool_list('non-free'), Path(output_path, 'api', 'non-free'))
 
 
 def register():