Merge pull request #3744 from mlibrary/HELIO-4820/kbart-remote-old-dirs

HELIO-4820 Always move old remote KBARTs of a group_key+"_old" dir

app/jobs/build_kbart_job.rb

@@ -10,7 +10,7 @@
 
 class BuildKbartJob < ApplicationJob
   def perform
-    MarcLogger.info("Beginning BuildKbartLog...")
+    MarcLogger.info("Beginning BuildKbartJob...")
 
     Greensub::Product.where(needs_kbart: true).each do |product|
       next if product.group_key.nil?
@@ -56,7 +56,7 @@ def perform
       sftp_kbart(new_kbart_name, product.group_key, file_root)
     end
 
-    MarcLogger.info("BuildKbartLog Finished")
+    MarcLogger.info("BuildKbartJob Finished")
   end
 
   def sftp_kbart(kbart, group_key, file_root)
@@ -81,16 +81,22 @@ def sftp_kbart(kbart, group_key, file_root)
 
   # HELIO-4531
   # fulcimen can only handle building marcs reliably if there is a single kbart per product
-  # Not all kbarts get marc files generated only UMPEBC, BAR, Amherst and Lever
+  # so move old ones out of the way. Do this for all products, not just the ones that fulcimen makes marc for
   def maybe_move_old_kbarts(sftp, group_key, file_root)
-    return unless ['aberdeen', 'umpebc', 'bar', 'amherst', 'leverpress', 'test_product'].include?(group_key)
-
     old_kbart_dir = if group_key == "umpebc"
                       File.join(Marc::DirectoryMapper.group_key_kbart[group_key], "UMPEBC_old")
                     else
                       File.join(Marc::DirectoryMapper.group_key_kbart[group_key], "#{group_key}_old")
                     end
 
+    begin
+      sftp.mkdir!(old_kbart_dir)
+    rescue Net::SFTP::StatusException, StandardError => e
+      # There's no way to check if a directory exists beforehand, so mkdir! can fail with an exception
+      # but it just means the directory already exists
+      MarcLogger.info("Ran mkdir '#{old_kbart_dir}' and got an exception (because the directory already exists?): #{e}")
+    end
+
     # You can't move, only "rename", https://stackoverflow.com/a/22260984 which makes this annoying
     sftp.dir.entries(Marc::DirectoryMapper.group_key_kbart[group_key]).each do |entry|
       match = entry.name.match(/#{file_root}_\d\d\d\d-\d\d-\d\d\.\w{3}$/)

spec/jobs/build_kbart_job_spec.rb

@@ -82,6 +82,7 @@
     before do
       allow_any_instance_of(BuildKbartJob).to receive(:yaml_config).and_return(login)
       allow(Net::SFTP).to receive(:start).and_yield(sftp)
+      allow(sftp).to receive(:mkdir!)
       allow(dir).to receive(:entries).and_return([])
     end
 
@@ -342,11 +343,33 @@
         allow(sftp.dir).to receive(:entries).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART").and_return(entries)
       end
 
-      it "renames (moves) the old kbarts to UMPEBC_old" do
-        subject.maybe_move_old_kbarts(sftp, group_key, file_root)
-        expect(sftp).to have_received(:rename).exactly(2).times
-        expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.csv", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.csv")
-        expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.txt", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.txt")
+      context "when the old kbart dir already exists" do
+        before do
+          allow(sftp).to receive(:mkdir!).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old").and_raise(StandardError)
+        end
+
+        it "throws on mkdir which is fine, renames (moves) the old kbarts to UMPEBC_old" do
+          subject.maybe_move_old_kbarts(sftp, group_key, file_root)
+          expect(sftp).to have_received(:mkdir!).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old")
+          expect(sftp).to have_received(:rename).exactly(2).times
+          expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.csv", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.csv")
+          expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.txt", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.txt")
+        end
+      end
+
+      context "when the old kbart diretory doesn't yet exist" do
+        before do
+          allow(sftp).to receive(:mkdir!).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old").and_return(true)
+          allow(MarcLogger).to receive(:info)
+        end
+
+        it "creates UMPEBC_old, renames (moves) the old kbarts to UMPEBC_old" do
+          subject.maybe_move_old_kbarts(sftp, group_key, file_root)
+          expect(sftp).to have_received(:mkdir!).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old")
+          expect(sftp).to have_received(:rename).exactly(2).times
+          expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.csv", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.csv")
+          expect(sftp).to have_received(:rename).with("/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_2018_2022-01-01.txt", "/home/fulcrum_ftp/ftp.fulcrum.org/UMPEBC/KBART/UMPEBC_old/UMPEBC_2018_2022-01-01.txt")
+        end
       end
     end
 
@@ -362,6 +385,7 @@
 
       before do
         allow(sftp.dir).to receive(:entries).with("/home/fulcrum_ftp/ftp.fulcrum.org/Amherst_College_Press/KBART").and_return(entries)
+        allow(sftp).to receive(:mkdir!).with("/home/fulcrum_ftp/ftp.fulcrum.org/Amherst_College_Press/KBART/amherst_old").and_return(true)
       end
 
       it "renames (moves) the old kbarts amherst_old" do