[GENE-2434] - Load vault token from file

.github/workflows/sonarqube.yml

@@ -0,0 +1,18 @@
+name: SonarQube
+
+on:
+  push:
+    branches:
+      - branch_8_3
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  sonarqube:
+    uses: Unity-Technologies/github-actions-workflows/.github/workflows/sonarqube.yml@main
+    with:
+      coverage: false

lucene/default-nested-ivy-settings.xml

@@ -33,7 +33,7 @@
 
   <resolvers>
     <ibiblio name="sonatype-releases" root="https://oss.sonatype.org/content/repositories/releases" m2compatible="true" />
-    <ibiblio name="maven.restlet.com" root="https://maven.restlet.com" m2compatible="true" />
+    <ibiblio name="maven.restlet.com" root="https://maven.restlet.talend.com" m2compatible="true" />
     <ibiblio name="releases.cloudera.com" root="https://repository.cloudera.com/artifactory/libs-release-local" m2compatible="true" />
 
     <filesystem name="local-maven-2" m2compatible="true" local="true">

lucene/expressions/ivy.xml

@@ -25,6 +25,9 @@
     <dependency org="org.antlr" name="antlr4-runtime" rev="${/org.antlr/antlr4-runtime}" conf="compile"/>
     <dependency org="org.ow2.asm" name="asm" rev="${/org.ow2.asm/asm}" conf="compile"/>
     <dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
+    <dependency org="org.springframework.vault" name="spring-vault-core" rev="${/org.springframework.vault/spring-vault-core}" conf="compile" />
+    <dependency org="org.springframework" name="spring-beans" rev="${/org.springframework/spring-beans}" conf="compile" />
+    <dependency org="org.springframework" name="spring-web" rev="${/org.springframework/spring-web}" conf="compile" />
     <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
   </dependencies>
 </ivy-module>

lucene/ivy-versions.properties

@@ -128,6 +128,7 @@ org.apache.calcite.version = 1.18.0
 /org.apache.commons/commons-collections4 = 4.2
 /org.apache.commons/commons-compress = 1.18
 /org.apache.commons/commons-configuration2 = 2.1.1
+/commons-configuration/commons-configuration = 1.10
 /org.apache.commons/commons-exec = 1.3
 /org.apache.commons/commons-lang3 = 3.8.1
 /org.apache.commons/commons-math3 = 3.6.1
@@ -325,3 +326,10 @@ ua.net.nlp.morfologik-ukrainian-search.version = 3.9.0
 /ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version}
 
 /xerces/xercesImpl = 2.9.1
+
+org.springframework.vault.version =2.3.2
+/org.springframework.vault/spring-vault-core = ${org.springframework.vault.version}
+
+org.springframework.spring.version = 5.0.12.RELEASE
+/org.springframework/spring-beans = ${org.springframework.spring.version}
+/org.springframework/spring-web = ${org.springframework.spring.version}

solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java

@@ -39,6 +39,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 /**
  * <p> {@link DocBuilder} is responsible for creating Solr documents out of the given configuration. It also maintains
@@ -83,6 +84,7 @@ public class DocBuilder {
   private DIHProperties propWriter;
   private DebugLogger debugLogger;
   private final RequestInfo reqParams;
+  private ObjectMapper mapper;
 
   public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
     INSTANCE.set(this);
@@ -98,6 +100,7 @@ public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties
     if (writer != null) {
       writer.init(ctx);
     }
+    this.mapper = new ObjectMapper();
   }
 
 
@@ -650,6 +653,23 @@ private void addFields(Entity entity, DocWrapper doc,
       Object value = entry.getValue();
       if (value == null)  continue;
       if (key.startsWith("$")) continue;
+        if (key.equalsIgnoreCase("payload")) {
+            if (value == null) {
+                continue;
+            }
+            try {
+                final SilkcloudIndexDocument document = (SilkcloudIndexDocument)mapper.readValue(value.toString(), (Class)SilkcloudIndexDocument.class);
+                for (final Map.Entry<String, Object> fieldEntry : document.getFields().entrySet()) {
+                    if (fieldEntry.getValue() != null) {
+                        doc.addField((String)fieldEntry.getKey(), fieldEntry.getValue());
+                    }
+                }
+                continue;
+            }
+            catch (Exception e) {
+                throw new DataImportHandlerException(500, e);
+            }
+        }
       Set<EntityField> field = entity.getColNameVsField().get(key);
       IndexSchema schema = null == reqParams.getRequest() ? null : reqParams.getRequest().getSchema();
       if (field == null && schema != null) {

solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileWatcher.java

@@ -0,0 +1,184 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright (C) 2014 SilkCloud and/or its affiliates. All rights reserved.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.nio.file.*;
+import java.util.ArrayList;
+import java.util.concurrent.ConcurrentHashMap;
+import static java.nio.file.StandardWatchEventKinds.*;
+
+/**
+ * Java doc.
+ */
+public class FileWatcher {
+    //region singleton
+    private static volatile FileWatcher instance;
+
+    public static FileWatcher getInstance() {
+        // lazy singleton
+        if (instance == null) {
+            synchronized (FileWatcher.class) {
+                if (instance == null) {
+                    FileWatcher instance = new FileWatcher();
+                    instance.initialize();
+                    FileWatcher.instance = instance;
+                }
+            }
+        }
+        return instance;
+    }
+
+    public static void setInstance(FileWatcher watcher) {
+        // setInstance for unit testing
+        instance = watcher;
+    }
+    //endregion
+
+    //region private fields
+    private static Logger logger = LoggerFactory.getLogger(FileWatcher.class);
+
+    private Thread thread;
+    private WatchService watchService;
+    private FileChangeListenerMap listeners = new FileChangeListenerMap();
+    //endregion
+
+    /**.
+     * File listener
+     */
+    public interface FileListener {
+        void onFileChanged(Path path, WatchEvent.Kind<Path> kind);
+    }
+
+    public FileWatcher addListener(Path path, FileListener listener) {
+        try {
+            WatchKey key = path.register(watchService, ENTRY_CREATE, ENTRY_MODIFY);
+            listeners.addListener(key, listener);
+        }
+        catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+
+        return this;
+    }
+
+    public void close() {
+        if (thread != null) {
+            thread.interrupt();
+            thread = null;
+        }
+    }
+
+    //region private methods
+
+    // protected empty constructor for subclassing/mocking
+    protected FileWatcher() {
+    }
+
+    private void initialize() {
+        try {
+            watchService = FileSystems.getDefault().newWatchService();
+        }
+        catch (IOException ex) {
+            throw new RuntimeException("Error creating watch service.", ex);
+        }
+        thread = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                FileWatcher.this.run();
+            }
+        });
+        thread.start();
+    }
+
+    private void run() {
+        for (; ; ) {
+            try {
+                WatchKey key = watchService.take();
+                Path path = (Path) key.watchable();
+
+                for (WatchEvent<?> event : key.pollEvents()) {
+                    WatchEvent.Kind<?> kind = event.kind();
+
+                    if (kind == StandardWatchEventKinds.OVERFLOW) {
+                        logger.warn("Overflow happened in FileWatcher for key " + key);
+                        continue;
+                    }
+
+                    WatchEvent<Path> ev = (WatchEvent<Path>) event;
+                    listeners.notify(key, path, ev.kind());
+                }
+            }
+            catch (InterruptedException ex) {
+                return;
+            }
+            catch (Exception ex) {
+                logger.warn("Error occurred in file watcher. ", ex);
+            }
+        }
+    }
+
+    //endregion
+
+    //region private classes
+
+    private static final class FileChangeListenerList {
+        private ArrayList<WeakReference<FileListener>> listeners = new ArrayList<>();
+
+        public void addListener(FileListener listener) {
+            listeners.add(new WeakReference<>(listener));
+        }
+
+        public void notify(Path path, WatchEvent.Kind<Path> kind) {
+            ArrayList<WeakReference<FileListener>> toRemove = new ArrayList<>();
+            for (WeakReference<FileListener> listenerWeakReference : listeners) {
+                FileListener listener = listenerWeakReference.get();
+                if (listener == null) {
+                    toRemove.add(listenerWeakReference);
+                }
+                else {
+                    try {
+                        listener.onFileChanged(path, kind);
+                    }
+                    catch (Exception ex) {
+                        logger.warn("Error occurred in file watcher. ", ex);
+                    }
+                }
+            }
+            listeners.removeAll(toRemove);
+        }
+    }
+
+    private static final class FileChangeListenerMap {
+        private ConcurrentHashMap<WatchKey, FileChangeListenerList> listeners = new ConcurrentHashMap<>();
+
+        public void addListener(WatchKey watchKey, FileListener listener) {
+            FileChangeListenerList list = listeners.get(watchKey);
+            if (list == null) {
+                final FileChangeListenerList newList = new FileChangeListenerList();
+                list = listeners.putIfAbsent(watchKey, newList);
+                if (list == null) {
+                    list = newList;
+                }
+            }
+            list.addListener(listener);
+        }
+
+        public void notify(WatchKey watchKey, Path path, WatchEvent.Kind<Path> kind) {
+            FileChangeListenerList list = listeners.get(watchKey);
+            if (list != null) {
+                list.notify(path, kind);
+            }
+        }
+    }
+
+    //endregion
+}

solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java

@@ -16,18 +16,13 @@
  */
 package org.apache.solr.handler.dataimport;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
 import org.apache.solr.common.SolrException;
 import org.apache.solr.util.CryptoKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.naming.InitialContext;
 import javax.naming.NamingException;
-
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -39,6 +34,32 @@
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
+import java.math.BigInteger;
+import java.math.BigDecimal;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.sql.ResultSetMetaData;
+import javax.naming.NamingException;
+import javax.naming.InitialContext;
+import java.util.concurrent.TimeUnit;
+import java.sql.SQLException;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.util.Properties;
+import java.util.HashMap;
+import java.sql.Connection;
+import java.util.concurrent.Callable;
+import org.slf4j.Logger;
+import java.util.Map;
+import java.util.Iterator;
+import org.apache.solr.handler.dataimport.VaultServiceImpl;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
 
 /**
  * <p> A DataSource implementation which can fetch data using JDBC. </p> <p> Refer to <a
@@ -267,11 +288,25 @@ private Connection getFromJndi(final Properties initProps, final String jndiName
   }
 
   private void resolveVariables(Context ctx, Properties initProps) {
-    for (Map.Entry<Object, Object> entry : initProps.entrySet()) {
-      if (entry.getValue() != null) {
-        entry.setValue(ctx.replaceTokens((String) entry.getValue()));
+      final Map<Object, Object> encryptedMap = new HashMap<Object, Object>();
+      VaultServiceImpl vaultService = new VaultServiceImpl();
+      Properties properties = vaultService.readVaultProperties();
+      for (final Map.Entry<Object, Object> entry : initProps.entrySet()) {
+          if (entry.getValue() != null) {
+              final String value = ctx.replaceTokens((String)entry.getValue());
+              if (entry.getKey().toString().endsWith(".vaultPath")) {
+                  final String key = entry.getKey().toString();
+                  encryptedMap.put(key.substring(0, key.lastIndexOf(".")), properties.getProperty(value));
+              }
+              else {
+                  entry.setValue(value);
+              }
+          }
       }
-    }
+      for (final Object key2 : encryptedMap.keySet()) {
+          initProps.remove(key2 + ".vaultPath");
+      }
+      initProps.putAll(encryptedMap);
   }
 
   @Override