From e4d466010de921279f5a43b4fe58dbd172768277 Mon Sep 17 00:00:00 2001 From: svteb Date: Thu, 27 Feb 2025 09:32:41 +0000 Subject: [PATCH] Feat: Migrate tar shard Refs: #2213 - Migrates tar shard to main testsuite project. - Additionally migrates/removes the find shard, the Find module only has one method and it is only used in tar spec tests. I've thus decided to move it into tar_spec.cr, this change can be reversed in the future if necessary. - Tar spec tests are intentionally not runnable, as there is currently a conflict between the lib and src/modules (duplicate declarations). Signed-off-by: svteb --- spec/modules/tar_spec.cr | 71 +++++++++++++++++++++++ spec/spec_helper.cr | 1 + src/modules/tar/tar.cr | 120 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 spec/modules/tar_spec.cr create mode 100644 src/modules/tar/tar.cr diff --git a/spec/modules/tar_spec.cr b/spec/modules/tar_spec.cr new file mode 100644 index 000000000..1f0791d10 --- /dev/null +++ b/spec/modules/tar_spec.cr @@ -0,0 +1,71 @@ +require "../spec_helper.cr" +require "file_utils" + +TAR_SPEC_DIR="./tar_spec_dir" + +def self.find(directory, wildcard="*.tar*", maxdepth="1", silent=true) + Log.info { "find command: find #{directory} -maxdepth #{maxdepth} -name #{wildcard}" } + status = Process.run("find #{directory} -maxdepth #{maxdepth} -name \"#{wildcard}\"", + shell: true, + output: output = IO::Memory.new, + error: stderr = IO::Memory.new) + found_files = output.to_s.split("\n").select{|x| x.empty? == false} + if found_files.size == 0 && !silent + raise "No files found!" + end + found_files +end + +describe "TarClient" do + before_all do + Dir.mkdir(TAR_SPEC_DIR) + end + + after_all do + Dir.delete(TAR_SPEC_DIR) + end + + it "'.tar' should tar a source file or directory", tags:["tar"] do + `rm #{TAR_SPEC_DIR}/test.tar` + TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "cnf-testsuite.yml") + (File.exists?("./spec/fixtures/cnf-testsuite.yml")).should be_true + ensure + `rm #{TAR_SPEC_DIR}/test.tar` + end + + it "'.untar' should untar a tar file into a directory", tags:["tar"] do + `rm #{TAR_SPEC_DIR}/test.tar` + TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "cnf-testsuite.yml") + TarClient.untar("#{TAR_SPEC_DIR}/test.tar", "#{TAR_SPEC_DIR}") + (File.exists?("#{TAR_SPEC_DIR}/cnf-testsuite.yml")).should be_true + ensure + `rm #{TAR_SPEC_DIR}/test.tar` + `rm #{TAR_SPEC_DIR}/cnf-testsuite.yml` + end + + it "'.modify_tar!' should untar file, yield to block, retar", tags:["tar"] do + `rm #{TAR_SPEC_DIR}/test.tar` + input_content = File.read("./spec/fixtures/litmus-operator-v1.13.2.yaml") + (input_content =~ /imagePullPolicy: Never/).should be_nil + TarClient.tar("#{TAR_SPEC_DIR}/test.tar", "./spec/fixtures", "litmus-operator-v1.13.2.yaml") + TarClient.modify_tar!("#{TAR_SPEC_DIR}/test.tar") do |directory| + template_files = Find.find(directory, "*.yaml*", "100") + Log.debug {"template_files: #{template_files}"} + template_files.map do |x| + input_content = File.read(x) + output_content = input_content.gsub(/(.*imagePullPolicy:)(.*)/,"\\1 Never") + + input_content = File.write(x, output_content) + end + end + + TarClient.untar("#{TAR_SPEC_DIR}/test.tar", "#{TAR_SPEC_DIR}") + (File.exists?("#{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml")).should be_true + input_content = File.read("#{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml") + (input_content =~ /imagePullPolicy: Never/).should_not be_nil + ensure + `rm #{TAR_SPEC_DIR}/test.tar` + `rm #{TAR_SPEC_DIR}/litmus-operator-v1.13.2.yaml` + end + +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 116f88bdd..218849f20 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -2,6 +2,7 @@ require "spec" require "colorize" require "../src/cnf_testsuite" require "../src/tasks/utils/utils.cr" +require "../src/modules/tar/tar.cr" ENV["CRYSTAL_ENV"] = "TEST" diff --git a/src/modules/tar/tar.cr b/src/modules/tar/tar.cr new file mode 100644 index 000000000..5a266e5ff --- /dev/null +++ b/src/modules/tar/tar.cr @@ -0,0 +1,120 @@ +require "colorize" +require "log" + +# todo put in a separate library. it shold go under ./tools for now +module TarClient + TAR_REPOSITORY_DIR = "/tmp/repositories" + TAR_MANIFEST_DIR = "/tmp/manifests" + TAR_DOWNLOAD_DIR = "/tmp/download" + TAR_IMAGES_DIR = "/tmp/images" + TAR_BIN_DIR = "/tmp/bin" + TAR_TMP_BASE = "/tmp" + TAR_MODIFY_DIR = "/tmp/modify_tar" + + module ShellCmd + def self.run(cmd, log_prefix, log_type="debug") + Log.info { "#{log_prefix} command: #{cmd}" } + status = Process.run( + cmd, + shell: true, + output: output = IO::Memory.new, + error: stderr = IO::Memory.new + ) + if log_type == "debug" + Log.debug { "#{log_prefix} output: #{output.to_s}" } + Log.debug { "#{log_prefix} stderr: #{stderr.to_s}" } + else + Log.info { "#{log_prefix} output: #{output.to_s}" } + Log.info { "#{log_prefix} stderr: #{stderr.to_s}" } + end + {status: status, output: output.to_s, error: stderr.to_s} + end + end + + def self.tar(tarball_name, working_directory, source_file_or_directory, options="") + Log.info { "cding into #{working_directory} and tarring #{source_file_or_directory} into #{tarball_name}" } + Log.info { "#{tarball_name} exists?: #{File.exists?(tarball_name)}" } + if File.exists?(tarball_name) + ShellCmd.run("tar -tvf #{tarball_name}", "#{tarball_name} contents before", log_type="info") + end + + cmd = "tar #{options} -czvf #{tarball_name} -C #{working_directory} #{source_file_or_directory}" + Log.info { "TarClient.tar command: #{cmd}" } + result = ShellCmd.run(cmd, "TarClient.tar", log_type="info") + + ShellCmd.run("tar -tvf #{tarball_name}", "#{tarball_name} contents after", log_type="info") + result + end + + def self.append(tarball_name, working_directory, source_file_or_directory, options="") + #working_directory: directory to cd into before running the tar command + Log.info { "cding into #{working_directory} and tarring #{source_file_or_directory} into #{tarball_name}" } + Log.info { "#{tarball_name} exists?: #{File.exists?(tarball_name)}" } + if File.exists?(tarball_name) + Log.info { "#{tarball_name} contents (before tar): #{`tar -tvf #{tarball_name}`}" } + end + + cmd = "tar #{options} -rf #{tarball_name} -C #{working_directory} #{source_file_or_directory}" + Log.info { "TarClient.tar (append) command: #{cmd}" } + status = Process.run(cmd, + shell: true, + output: output = IO::Memory.new, + error: stderr = IO::Memory.new) + Log.info { "TarClient.tar output: #{output.to_s}" } + Log.info { "TarClient.tar stderr: #{stderr.to_s}" } + Log.info { "#{tarball_name} contents (after tar): #{`tar -tvf #{tarball_name}`}" } + {status: status, output: output, error: stderr} + end + + def self.untar(tarball_name, destination_directory, options="") + cmd = "tar #{options} -xvf #{tarball_name} -C #{destination_directory}" + Log.info { "TarClient.untar command: #{cmd}" } + status = Process.run(cmd, + shell: true, + output: output = IO::Memory.new, + error: stderr = IO::Memory.new) + Log.info { "TarClient.untar output: #{output.to_s}" } + Log.info { "TarClient.untar stderr: #{stderr.to_s}" } + {status: status, output: output, error: stderr} + end + + # + # modify_tar! << untars file, yields to block, retars, keep in tar module + # + def self.modify_tar!(tar_file) + raise "Critical Error" if tar_file.empty? || tar_file == '/' + tar_name = tar_file.split("/")[-1] + FileUtils.mkdir_p(TAR_MODIFY_DIR) + TarClient.untar(tar_file, TAR_MODIFY_DIR) + yield TAR_MODIFY_DIR + FileUtils.rm_rf(tar_file) + file_list = `ls #{TAR_MODIFY_DIR}`.gsub("\n", " ") + TarClient.tar(tar_file, TAR_MODIFY_DIR, file_list) + FileUtils.rm_rf(TAR_MODIFY_DIR) + end + + + def self.tar_file_by_url(url, append_file : String = "./downloaded.tar.gz", output_file="") + download_path = "download/" + FileUtils.rm_rf("/tmp/#{download_path}") + FileUtils.mkdir_p("/tmp/" + download_path) + if output_file.empty? + download_name = url.split("/").last + else + download_name = output_file + end + download_full_path = download_path + download_name + Log.info { "download_name: #{download_name}" } + Log.info { "download_full_path: #{download_full_path}" } + Halite.get("#{url}") do |response| + File.open("/tmp/" + download_full_path, "w") do |file| + IO.copy(response.body_io, file) + end + end + TarClient.append(append_file, "/tmp", download_full_path) + ensure + FileUtils.rm_rf("/tmp/#{download_path}") + end + +end +