Prepare for dss_python_backend

Author: PMeira

.github/workflows/builds.yml

@@ -25,7 +25,6 @@ jobs:
       image: ${{ matrix.container-image }}
     env:
       CONDA_SUBDIR: 'linux-64'
-      DSS_CAPI_PATH: '${{ github.workspace }}/dss_capi'
       CONDA: "/opt/miniconda/"
     steps: 
       - name: 'Checkout'
@@ -35,9 +34,7 @@ jobs:
           git checkout $GITHUB_SHA
       - name: 'Download/extract DSS C-API binaries'
         run: |
-          curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/dss_capi_${DSS_CAPI_TAG}_linux_x64.tar.gz -o dss_capi.tar.gz
           curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/messages.tar.gz -o messages.tar.gz
-          tar zxf dss_capi.tar.gz
           cd dss_python/dss
           tar zxf ../../messages.tar.gz
       - name: Build wheel
@@ -74,16 +71,14 @@ jobs:
           docker pull $DOCKER_IMAGE
       - name: 'Download/extract DSS C-API binaries'
         run: |
-          curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/dss_capi_${DSS_CAPI_TAG}_linux_x86.tar.gz -o dss_capi.tar.gz
           curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/messages.tar.gz -o messages.tar.gz
-          tar zxf dss_capi.tar.gz
           cd dss_python/dss
           tar zxf ../../messages.tar.gz
       - name: Build wheel
         run: |
           mkdir -p artifacts
           mkdir -p artifacts_raw
-          docker run -e GITHUB_SHA -e GITHUB_REF -e DSS_CAPI_PATH=/build/dss_capi -v "${PWD}:/build" -w /build $DOCKER_IMAGE bash /build/dss_python/ci/build_linux.sh x86
+          docker run -e GITHUB_SHA -e GITHUB_REF -v "${PWD}:/build" -w /build $DOCKER_IMAGE bash /build/dss_python/ci/build_linux.sh x86
       - name: 'Upload artifacts'
         uses: "actions/upload-artifact@v2"
         with:
@@ -94,7 +89,6 @@ jobs:
     name: 'macOS x64'
     runs-on: 'macos-10.15'
     env:
-      DSS_CAPI_PATH: '${{ github.workspace }}/dss_capi'
       SDKROOT: '${{ github.workspace }}/MacOSX10.13.sdk'
       PYTHON: python3
     steps:
@@ -104,15 +98,9 @@ jobs:
           path: 'dss_python'
       - name: 'Download/extract DSS C-API binaries'
         run: |
-          curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/dss_capi_${DSS_CAPI_TAG}_darwin_x64.tar.gz -o dss_capi.tar.gz
           curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/messages.tar.gz -o messages.tar.gz
-          tar zxf dss_capi.tar.gz
           cd dss_python/dss
           tar zxf ../../messages.tar.gz
-      - name: 'Download macOS SDK 10.13'
-        run: |
-          curl -s -L https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.13.sdk.tar.xz -o macOSsdk.tar.xz
-          tar xf macOSsdk.tar.xz
       - name: Build wheel
         run: |
           bash dss_python/ci/build_wheel.sh
@@ -131,8 +119,6 @@ jobs:
     name: 'macOS ARM64'
     runs-on: 'macos-11'
     env:
-      DSS_CAPI_PATH: '${{ github.workspace }}/dss_capi'
-      # SDKROOT: '${{ github.workspace }}/MacOSX10.13.sdk'
       PYTHON: python3
       _PYTHON_HOST_PLATFORM: macosx-11.0-arm64
       ARCHFLAGS: '-arch arm64'
@@ -144,9 +130,7 @@ jobs:
           path: 'dss_python'
       - name: 'Download/extract DSS C-API binaries'
         run: |
-          curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/dss_capi_${DSS_CAPI_TAG}_darwin_arm64.tar.gz -o dss_capi.tar.gz
           curl -s -L https://github.com/dss-extensions/dss_capi/releases/download/${DSS_CAPI_TAG}/messages.tar.gz -o messages.tar.gz
-          tar zxf dss_capi.tar.gz
           cd dss_python/dss
           tar zxf ../../messages.tar.gz
       # - name: 'Download macOS SDK 10.13'
@@ -172,7 +156,6 @@ jobs:
     runs-on: windows-2019
     env:
       CONDA_SUBDIR: 'win-64'
-      DSS_CAPI_PATH: '${{ github.workspace }}\dss_capi'
       PYTHON: python
     steps:
       - uses: actions/checkout@v2
@@ -182,9 +165,7 @@ jobs:
       - name: 'Download/extract DSS C-API binaries'
         shell: cmd
         run: |
-          "c:\Program Files\Git\mingw64\bin\curl" -s -L https://github.com/dss-extensions/dss_capi/releases/download/%DSS_CAPI_TAG%/dss_capi_%DSS_CAPI_TAG%_win_x64.zip -o dss_capi.zip
           "c:\Program Files\Git\mingw64\bin\curl" -s -L https://github.com/dss-extensions/dss_capi/releases/download/%DSS_CAPI_TAG%/messages.zip -o messages.zip
-          7z x -y -o. dss_capi.zip
           cd dss_python\dss
           tar zxf ..\..\messages.zip
       - name: Build wheel
@@ -207,7 +188,6 @@ jobs:
     runs-on: windows-2019
     env:
       CONDA_SUBDIR: 'win-32'
-      DSS_CAPI_PATH: '${{ github.workspace }}\dss_capi'
       PYTHON: python
     steps:
       - uses: actions/checkout@v2
@@ -221,9 +201,7 @@ jobs:
       - name: 'Download/extract DSS C-API binaries'
         shell: cmd
         run: |
-          "c:\Program Files\Git\mingw64\bin\curl" -s -L https://github.com/dss-extensions/dss_capi/releases/download/%DSS_CAPI_TAG%/dss_capi_%DSS_CAPI_TAG%_win_x86.zip -o dss_capi.zip
           "c:\Program Files\Git\mingw64\bin\curl" -s -L https://github.com/dss-extensions/dss_capi/releases/download/%DSS_CAPI_TAG%/messages.zip -o messages.zip
-          7z x -y -o. dss_capi.zip
           cd dss_python\dss
           tar zxf ..\..\messages.zip
       - name: Build wheel

README.md

@@ -16,7 +16,7 @@ Please see [FAQ](https://github.com/dss-extensions/dss-extensions#faq) for a cou
 - [DSS Sharp](http://github.com/dss-extensions/dss_sharp/) is available for .NET/C#, [packaged on NuGet](https://www.nuget.org/packages/dss_sharp/), also mimics the COM classes (drop-in replacement for `OpenDSSengine.DLL`). The current version is now multi-platform too! Soon it will be possible to use it via COM.
 - [DSS MATLAB](http://github.com/dss-extensions/dss_matlab/) presents multi-platform integration (Windows, Linux, MacOS) with DSS C-API and is also very compatible with the API of the official OpenDSS COM classes.
 
-Version 0.13.x is based on OpenDSS revision 3595 (few commits after OpenDSS v9.6.1.1, but we pick and choose when to port certain features). While we plan to add a lot more functionality into DSS Python, the main goal of creating a COM-compatible API has been reached in 2018. If you find an unexpected missing feature, please report it! Currently missing features that will be implemented eventually are plotting (partial implementation available) and diakoptics (planned for future version).
+Version 0.14.x is based on OpenDSS revision 3595 (few commits after OpenDSS v9.6.1.1, but we pick and choose when to port certain features). While we plan to add a lot more functionality into DSS Python, the main goal of creating a COM-compatible API has been reached in 2018. If you find an unexpected missing feature, please report it! Currently missing features that will be implemented eventually are plotting (partial implementation available) and diakoptics (planned for future version).
 
 This module mimics the COM structure (as exposed via `win32com` or `comtypes`) -- see [The DSS instance](https://dss-extensions.org/dss_python/#the-dss-instance) for some docs --  effectively enabling multi-platform compatibility at Python level. Compared to other options, it provides easier migration from code that uses the official OpenDSS through COM.
 Most of the COM documentation can be used as-is, but instead of returning tuples or lists, this modules returns/accepts NumPy arrays for numeric data exchange, which is usually preferred by the users.
@@ -25,6 +25,7 @@ The module depends on CFFI, NumPy and, optionally, SciPy.Sparse for reading the
 
 ## Brief release history
 
+- 2023-04-03 / version 0.14.0: Same as 0.13.1, but splits dss_python into two packages: [dss_python_backend](https://github.com/dss-extensions/dss_python_backend) now contains the native binaries/DLLs. No need for user action.
 - **2023-04-01 / version 0.13.1: Microupdate to the OpenDSS engine; very minor Python changes.**
 - 2023-03-29 / version 0.13.0: Updates to the OpenDSS engine, move plotting and initial notebook integration, matrix shapes and complex numbers, bug fixes, etc.
 - 2022-07-16 / version 0.12.1: Very minor release to address a bug found in v0.12.0, to add the Storages API, and include the property descriptions/help.

ci/build_conda.sh

@@ -6,7 +6,6 @@ if [ "$RUNNER_OS" = "Windows" ]; then
 fi
 
 export PATH=$CONDA:$CONDA/bin:$CONDA/scripts:$PATH
-export DSS_PYTHON_MANYLINUX=0
 
 source $CONDA/etc/profile.d/conda.sh
 

ci/build_linux.sh

@@ -1,14 +1,7 @@
 set -e -x
 export PATH=/opt/python/cp39-cp39/bin/:$PATH
-export DSS_PYTHON_MANYLINUX=1
 
 cd dss_python
-python3 -m pip install --upgrade pip cffi wheel
-python3 setup.py --quiet bdist_wheel --py-limited-api cp37 --dist-dir="../artifacts_raw"
+python3 -m pip install --upgrade pip wheel
+python3 setup.py --quiet bdist_wheel --py-limited-api cp37 --dist-dir="../artifacts"
 cd ..
-
-# Bundle external shared libraries into the wheels
-export LD_LIBRARY_PATH=${DSS_CAPI_PATH}/lib/linux_$1:$LD_LIBRARY_PATH
-for whl in artifacts_raw/*.whl; do
-    auditwheel repair "$whl" -w artifacts
-done

docs/changelog.md

@@ -1,3 +1,11 @@
+# 0.14.x
+
+## 0.14.0
+
+Same as version 0.13.1, except that the organization of the module has changed to facilitate fast iteration of the Python side of the code.
+
+We introduced the new package `dss_python_backend` that will contain all native libs and CFFI bindings from this point on. This leaves DSS-Python itself as a pure Python package, which is much easier to package and distribute.
+
 # 0.13.x
 
 ## 0.13.1

docs/examples/UserModels/PyIndMach012/README.ipynb

@@ -1,5 +1,5 @@
 import re
-from .. import (
+from dss_python_backend import (
     # _dss_CapUserControl, 
     _dss_GenUserModel, 
     # _dss_PVSystemUserModel, 

dss/UserModels/wrappers.py

@@ -5,21 +5,13 @@
 This module used to provide instances for the OpenDSS Version 7 implementation. 
 As of 2022, most of the parallel-machine functions of EPRI's OpenDSS have been reimplemented using a different approach. Therefore the PM functions are available in the instances of this module too.
 Besides the parallel-machine mechanisms, DSS Python also exposes the DSSContext mechanism provided by DSS Extensions. DSSContexts allow using multiple OpenDSS instances directly, including user-managed multi-threading, without using the internal OpenDSS actors.
+
+Starting April 2023, our CFFI extension modules and DSS C-API binaries moved to `dss_python_backend` and are justed consumed here.
 '''
 
 import os
 from .patch_dss_com import patch_dss_com
-
-if os.environ.get('DSS_EXTENSIONS_DEBUG', '') != '1':
-    from ._dss_capi import ffi, lib
-else:
-    import warnings
-    warnings.warn('Environment variable DSS_EXTENSIONS_DEBUG=1 is set: loading the debug version of the DSS C-API library')
-    from ._dss_capid import ffi, lib
-
-# Ensure this is called at least once. This was moved from 
-# CffiApiUtil so we call it as soon as the DLL/so is loaded.
-lib.DSS_Start(0)
+from dss_python_backend import ffi, lib
 _properties_mo = os.path.join(os.path.dirname(__file__), 'messages', 'properties-en-US.mo')
 if os.path.exists(_properties_mo):
     lib.DSS_SetPropertiesMO(_properties_mo.encode())
@@ -50,5 +42,5 @@
 dss = DSS = DSS_GR #: Same as DSS_GR
 
 
-__version__ = '0.13.2.dev'
+__version__ = '0.14.0.dev'
 __all__ = ['dss', 'DSS', 'DSS_GR', 'prime_api_util', 'api_util', 'DSSException', 'patch_dss_com', 'set_case_insensitive_attributes', 'enums', 'Edit']