Ignore files according to the patterns declared in ".gitignore" when using pio package pack // Resolve #4188

Author: ivankravets

HISTORY.rst

@@ -19,6 +19,7 @@ PlatformIO Core 5
     * `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`_ - run command from package tool (`issue #4163 <https://github.com/platformio/platformio-core/issues/4163>`_)
 
   - Added support for dependencies declared in a "tool" type package
+  - Ignore files according to the patterns declared in ".gitignore" when using `pio package pack <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ command (`issue #4188 <https://github.com/platformio/platformio-core/issues/4188>`_)
   - Dropped automatic updates of global libraries and development platforms (`issue #4179 <https://github.com/platformio/platformio-core/issues/4179>`_)
   - Dropped support for "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies <https://docs.platformio.org/en/latest/scripting/examples/extra_python_packages.html>`__
 

docs

@@ -33,7 +33,12 @@
 
 
 class PackagePacker(object):
-    INCLUDE_DEFAULT = ManifestFileType.items().values()
+    INCLUDE_DEFAULT = list(ManifestFileType.items().values()) + [
+        "README",
+        "README.md",
+        "README.rst",
+        "LICENSE",
+    ]
     EXCLUDE_DEFAULT = [
         # PlatformIO internal files
         PackageItem.METAFILE_NAME,
@@ -125,6 +130,20 @@ def get_archive_name(name, version, system=None):
             ),
         )
 
+    @staticmethod
+    def load_gitignore_filters(path):
+        result = []
+        with open(path, encoding="utf8") as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if not line or line.startswith(("#")):
+                    continue
+                if line.startswith("!"):
+                    result.append(f"+<{line[1:]}>")
+                else:
+                    result.append(f"-<{line}>")
+        return result
+
     def pack(self, dst=None):
         tmp_dir = tempfile.mkdtemp()
         try:
@@ -156,7 +175,7 @@ def pack(self, dst=None):
             elif os.path.isdir(dst):
                 dst = os.path.join(dst, filename)
 
-            return self._create_tarball(src, dst, manifest)
+            return self.create_tarball(src, dst, manifest)
         finally:
             shutil.rmtree(tmp_dir)
 
@@ -183,7 +202,7 @@ def find_source_root(self, src):
 
         return src
 
-    def _create_tarball(self, src, dst, manifest):
+    def create_tarball(self, src, dst, manifest):
         include = manifest.get("export", {}).get("include")
         exclude = manifest.get("export", {}).get("exclude")
         # remap root
@@ -224,11 +243,15 @@ def compute_src_filters(self, src, include, exclude):
         result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT]
         # exclude items declared in manifest
         result += ["-<%s>" % p for p in exclude or []]
+
         # apply extra excludes if no custom "export" field in manifest
         if (not include and not exclude) or isinstance(
             self.manifest_parser, LibraryPropertiesManifestParser
         ):
             result += ["-<%s>" % p for p in exclude_extra]
-        # automatically include manifests
+            if os.path.exists(os.path.join(src, ".gitignore")):
+                result += self.load_gitignore_filters(os.path.join(src, ".gitignore"))
+
+        # always include manifests and relevant files
         result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT]
         return result

examples

@@ -27,8 +27,8 @@
 def test_base(tmpdir_factory):
     pkg_dir = tmpdir_factory.mktemp("package")
     pkg_dir.join(".git").mkdir().join("file").write("")
-    pkg_dir.join(".gitignore").write("tests")
-    pkg_dir.join("._ignored").write("")
+    pkg_dir.join(".gitignore").write("")
+    pkg_dir.join("._hidden_file").write("")
     pkg_dir.join("main.cpp").write("#include <stdio.h>")
     p = PackagePacker(str(pkg_dir))
     # test missed manifest
@@ -95,6 +95,42 @@ def test_filters(tmpdir_factory):
         )
 
 
+def test_gitgnore_filters(tmpdir_factory):
+    pkg_dir = tmpdir_factory.mktemp("package")
+    pkg_dir.join(".git").mkdir().join("file").write("")
+    pkg_dir.join(".gitignore").write(
+        """
+# comment
+
+gi_file
+gi_folder
+gi_folder_*
+
+**/main_nested.h
+
+gi_keep_file
+!gi_keep_file
+LICENSE
+"""
+    )
+    pkg_dir.join("LICENSE").write("")
+    pkg_dir.join("gi_keep_file").write("")
+    pkg_dir.join("gi_file").write("")
+    pkg_dir.mkdir("gi_folder").join("main.h").write("#ifndef")
+    pkg_dir.mkdir("gi_folder_name").join("main.h").write("#ifndef")
+    pkg_dir.mkdir("gi_nested_folder").mkdir("a").mkdir("b").join("main_nested.h").write(
+        "#ifndef"
+    )
+    pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}')
+    p = PackagePacker(str(pkg_dir))
+    with fs.cd(str(pkg_dir)):
+        p.pack()
+    with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar:
+        assert set(tar.getnames()) == set(
+            ["library.json", "LICENSE", ".gitignore", "gi_keep_file"]
+        )
+
+
 def test_symlinks(tmpdir_factory):
     # Windows does not support symbolic links
     if IS_WINDOWS: