Shard migration: Migrate cnf-testsuite org shards to main testsuite

spec/modules/tar_spec.cr

@@ -0,0 +1,71 @@
+require "../spec_helper.cr"
+require "file_utils"
+
+TAR_SPEC_DIR="./tar_spec_dir"
+
+def self.find(directory, wildcard="*.tar*", maxdepth="1", silent=true)
+  Log.info { "find command: find #{directory} -maxdepth #{maxdepth} -name #{wildcard}" }
+  status = Process.run("find #{directory} -maxdepth #{maxdepth} -name \"#{wildcard}\"",
+                       shell: true,
+                       output: output = IO::Memory.new,
+                       error: stderr = IO::Memory.new)
+  found_files = output.to_s.split("\n").select{|x| x.empty? == false}
+  if found_files.size == 0 && !silent
+    raise "No files found!"
+  end
+  found_files
+end
+
+describe "TarClient" do
+  before_all do
+    Dir.mkdir(TAR_SPEC_DIR)
+  end
+
+  after_all do
+    Dir.delete(TAR_SPEC_DIR)
+  end
+
+  it "'.tar' should tar a source file or directory", tags:["tar"]  do
+    `rm #{TAR_SPEC_DIR}/test.tar`
+    TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "cnf-testsuite.yml")
+    (File.exists?("./spec/fixtures/cnf-testsuite.yml")).should be_true
+  ensure
+    `rm #{TAR_SPEC_DIR}/test.tar`
+  end
+
+  it "'.untar' should untar a tar file into a directory", tags:["tar"]  do
+    `rm #{TAR_SPEC_DIR}/test.tar`
+    TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "cnf-testsuite.yml")
+    TarClient.untar("#{TAR_SPEC_DIR}/test.tar", "#{TAR_SPEC_DIR}")
+    (File.exists?("#{TAR_SPEC_DIR}/cnf-testsuite.yml")).should be_true
+  ensure
+    `rm #{TAR_SPEC_DIR}/test.tar`
+    `rm #{TAR_SPEC_DIR}/cnf-testsuite.yml`
+  end
+
+  it "'.modify_tar!' should untar file, yield to block, retar", tags:["tar"]  do
+    `rm #{TAR_SPEC_DIR}/test.tar`
+    input_content = File.read("./spec/fixtures/litmus-operator-v1.13.2.yaml") 
+    (input_content =~ /imagePullPolicy: Never/).should be_nil
+    TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "litmus-operator-v1.13.2.yaml")
+    TarClient.modify_tar!("#{TAR_SPEC_DIR}/test.tar") do |directory| 
+      template_files = Find.find(directory, "*.yaml*", "100")
+      Log.debug {"template_files: #{template_files}"}
+      template_files.map do |x| 
+        input_content = File.read(x) 
+        output_content = input_content.gsub(/(.*imagePullPolicy:)(.*)/,"\\1 Never")
+
+        input_content = File.write(x, output_content) 
+      end
+    end
+
+    TarClient.untar("#{TAR_SPEC_DIR}/test.tar", "#{TAR_SPEC_DIR}")
+    (File.exists?("#{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml")).should be_true
+    input_content = File.read("#{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml") 
+    (input_content =~ /imagePullPolicy: Never/).should_not be_nil
+  ensure
+    `rm #{TAR_SPEC_DIR}/test.tar`
+    `rm #{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml`
+  end
+
+end

spec/spec_helper.cr

@@ -2,6 +2,7 @@ require "spec"
 require "colorize"
 require "../src/cnf_testsuite"
 require "../src/tasks/utils/utils.cr"
+require "../src/modules/tar/tar.cr"
 
 ENV["CRYSTAL_ENV"] = "TEST" 
 

src/modules/tar/tar.cr

@@ -0,0 +1,120 @@
+require "colorize"
+require "log"
+
+# todo put in a separate library. it shold go under ./tools for now
+module TarClient
+  TAR_REPOSITORY_DIR = "/tmp/repositories"
+  TAR_MANIFEST_DIR = "/tmp/manifests"
+  TAR_DOWNLOAD_DIR = "/tmp/download"
+  TAR_IMAGES_DIR = "/tmp/images"
+  TAR_BIN_DIR = "/tmp/bin"
+  TAR_TMP_BASE = "/tmp"
+  TAR_MODIFY_DIR = "/tmp/modify_tar"
+
+  module ShellCmd
+    def self.run(cmd, log_prefix, log_type="debug")
+      Log.info { "#{log_prefix} command: #{cmd}" }
+      status = Process.run(
+        cmd,
+        shell: true,
+        output: output = IO::Memory.new,
+        error: stderr = IO::Memory.new
+      )
+      if log_type == "debug"
+        Log.debug { "#{log_prefix} output: #{output.to_s}" }
+        Log.debug { "#{log_prefix} stderr: #{stderr.to_s}" }
+      else
+        Log.info { "#{log_prefix} output: #{output.to_s}" }
+        Log.info { "#{log_prefix} stderr: #{stderr.to_s}" }
+      end
+      {status: status, output: output.to_s, error: stderr.to_s}
+    end
+  end
+
+  def self.tar(tarball_name, working_directory, source_file_or_directory, options="")
+    Log.info { "cding into #{working_directory} and tarring #{source_file_or_directory} into #{tarball_name}" }
+    Log.info { "#{tarball_name} exists?: #{File.exists?(tarball_name)}" }
+    if File.exists?(tarball_name)
+      ShellCmd.run("tar -tvf #{tarball_name}", "#{tarball_name} contents before", log_type="info")
+    end
+
+    cmd = "tar #{options} -czvf #{tarball_name} -C #{working_directory} #{source_file_or_directory}"
+    Log.info { "TarClient.tar command: #{cmd}" }
+    result = ShellCmd.run(cmd, "TarClient.tar", log_type="info")
+
+    ShellCmd.run("tar -tvf #{tarball_name}", "#{tarball_name} contents after", log_type="info")
+    result
+  end
+
+  def self.append(tarball_name, working_directory, source_file_or_directory, options="")
+    #working_directory: directory to cd into before running the tar command
+    Log.info { "cding into #{working_directory} and tarring #{source_file_or_directory} into #{tarball_name}" }
+    Log.info { "#{tarball_name} exists?: #{File.exists?(tarball_name)}" }
+    if File.exists?(tarball_name)
+      Log.info { "#{tarball_name} contents (before tar): #{`tar -tvf #{tarball_name}`}" }
+    end
+
+    cmd = "tar #{options} -rf #{tarball_name} -C #{working_directory} #{source_file_or_directory}"
+    Log.info { "TarClient.tar (append) command: #{cmd}" }
+    status = Process.run(cmd,
+                         shell: true,
+                         output: output = IO::Memory.new,
+                         error: stderr = IO::Memory.new)
+    Log.info { "TarClient.tar output: #{output.to_s}" }
+    Log.info { "TarClient.tar stderr: #{stderr.to_s}" }
+    Log.info { "#{tarball_name} contents (after tar): #{`tar -tvf #{tarball_name}`}" }
+    {status: status, output: output, error: stderr}
+  end
+
+  def self.untar(tarball_name, destination_directory, options="")
+    cmd = "tar #{options} -xvf #{tarball_name} -C #{destination_directory}"
+    Log.info { "TarClient.untar command: #{cmd}" }
+    status = Process.run(cmd,
+                         shell: true,
+                         output: output = IO::Memory.new,
+                         error: stderr = IO::Memory.new)
+    Log.info { "TarClient.untar output: #{output.to_s}" }
+    Log.info { "TarClient.untar stderr: #{stderr.to_s}" }
+    {status: status, output: output, error: stderr}
+  end
+
+  #
+  # modify_tar! << untars file, yields to block, retars, keep in tar module
+  #
+  def self.modify_tar!(tar_file)
+    raise "Critical Error" if tar_file.empty? || tar_file == '/'
+    tar_name = tar_file.split("/")[-1]
+    FileUtils.mkdir_p(TAR_MODIFY_DIR)
+    TarClient.untar(tar_file, TAR_MODIFY_DIR)
+    yield TAR_MODIFY_DIR
+    FileUtils.rm_rf(tar_file)
+    file_list = `ls #{TAR_MODIFY_DIR}`.gsub("\n", " ") 
+    TarClient.tar(tar_file, TAR_MODIFY_DIR, file_list)
+    FileUtils.rm_rf(TAR_MODIFY_DIR)
+  end
+
+
+  def self.tar_file_by_url(url, append_file : String = "./downloaded.tar.gz", output_file="")
+    download_path = "download/" 
+    FileUtils.rm_rf("/tmp/#{download_path}")
+    FileUtils.mkdir_p("/tmp/" + download_path)
+    if output_file.empty?
+      download_name = url.split("/").last
+    else
+      download_name = output_file
+    end
+    download_full_path = download_path + download_name
+    Log.info { "download_name: #{download_name}" }
+    Log.info { "download_full_path: #{download_full_path}" }
+    Halite.get("#{url}") do |response| 
+       File.open("/tmp/" + download_full_path, "w") do |file| 
+         IO.copy(response.body_io, file)
+       end
+    end
+    TarClient.append(append_file, "/tmp", download_full_path)
+  ensure
+    FileUtils.rm_rf("/tmp/#{download_path}")
+  end
+
+end
+