Merge pull request #321 from centerforaisafety/320-backups-ldap

320 backups ldap

playbooks/roles/backups/files/ldap_backup.py

@@ -0,0 +1,152 @@
+"""
+Automates the LDAP backup process, including:
+- Extracting group, user, and association information.
+- Compressing the backup file.
+- Uploading the backup to Object Storage.
+
+Logging: Outputs logs to /opt/oci-hpc/logs/backups/backup_ldap.log
+
+Dependencies:
+- Python 3.x
+- `cluster` CLI tool for retrieving LDAP data
+- `oci` CLI tool for uploading to Object Storage
+"""
+
+import subprocess
+import re
+import json
+import logging
+from typing import Dict, List
+from datetime import datetime
+
+# Configure logging
+logging.basicConfig(
+    filename='/opt/oci-hpc/logs/backups/backup_ldap.log',
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
+def run_command(command: str) -> str:
+    """Executes a shell command and returns the output as a string."""
+    try:
+        process = subprocess.run(command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        return process.stdout
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Command '{command}' failed with error: {e.stderr}")
+        return ""
+
+# Extract group information
+def extract_group_info(output: str) -> Dict[str, str]:
+    """Parses the command output to extract group information."""
+    group_info = {}
+    lines = output.splitlines()
+
+    current_group_name = None
+    for line in lines:
+        line = line.strip()
+        if line.startswith('cn:'):
+            current_group_name = line.split(':')[1].strip()
+        elif line.startswith('gidNumber:'):
+            group_id = line.split(':')[1].strip()
+            if current_group_name:
+                group_info[current_group_name] = group_id
+    return group_info
+
+# Extract user information
+def extract_user_info(output: str) -> Dict[str, Dict[str, str]]:
+    """Parses the command output to extract user information."""
+    user_info = {}
+    user_blocks = re.findall(r"DN: cn=(.*?)(?=DN: cn=|$)", output, re.DOTALL)
+
+    for block in user_blocks:
+        uid = re.search(r"uid: (\S+)", block)
+        uid_number = re.search(r"uidNumber: (\d+)", block)
+        gid_number = re.search(r"gidNumber: (\d+)", block)
+        display_name = re.search(r"displayName: (.*)", block)
+
+        if uid and uid_number and gid_number:
+            user_info[uid.group(1)] = {
+                'uidNumber': uid_number.group(1),
+                'gidNumber': gid_number.group(1),
+                'displayName': display_name.group(1).strip() if display_name else "Unknown"
+            }
+    return user_info
+
+# Extract user-group associations
+def extract_users(output: str) -> List[str]:
+    """Extracts user names from the command output using regular expressions."""
+    user_entries = re.findall(r'uid:\s*(\w+)', output)
+    return user_entries
+
+def get_user_groups(user: str) -> List[str]:
+    """Retrieves the groups for a given user using the 'id -Gn' command."""
+    try:
+        result = subprocess.run(['id', '-Gn', user], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if result.returncode == 0:
+            return result.stdout.strip().split()
+        else:
+            logging.error(f"Error retrieving groups for user {user}: {result.stderr}")
+            return []
+    except Exception as e:
+        logging.error(f"An error occurred while retrieving groups for user {user}: {e}")
+        return []
+
+# Main function to generate, compress, and upload the backup
+def main():
+    """Main function to run commands, parse output, compress, and upload the backup."""
+    # Generate a timestamp for the backup file
+    timestamp = datetime.now().strftime('%Y_%m_%d')
+    output_file = f'/tmp/ldap_backup_{timestamp}.json'
+    compressed_file = f'{output_file}.gz'
+
+    logging.info("Starting LDAP backup process.")
+
+    # Extract group information
+    group_command = 'cluster group list'
+    group_output = run_command(group_command)
+    groups = extract_group_info(group_output)
+
+    # Extract user information
+    user_command = 'cluster user list'
+    user_output = run_command(user_command)
+    users = extract_user_info(user_output)
+
+    # Extract user-group associations
+    user_names = extract_users(user_output)
+    associations = {user: get_user_groups(user) for user in user_names}
+
+    # Combine all dictionaries into one JSON
+    combined_data = {
+        "groups": groups,
+        "users": users,
+        "associations": associations
+    }
+
+    # Write to JSON file
+    try:
+        with open(output_file, 'w') as json_file:
+            json.dump(combined_data, json_file, indent=4)
+    except IOError as e:
+        logging.error(f"Failed to write to {output_file}: {e}")
+        return
+
+    # Compress the JSON file
+    try:
+        subprocess.run(['gzip', output_file], check=True)
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Compression failed: {e}")
+        return
+
+    # Upload the compressed file to Object Storage
+    try:
+        bucket_name = 'backups'
+        oci_command = f'oci os object put --bucket-name {bucket_name} --file {compressed_file}'
+        subprocess.run(oci_command, shell=True, check=True)
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Upload failed: {e}")
+
+    logging.info("LDAP backup process completed.")
+
+# Run the main function
+if __name__ == '__main__':
+    main()

playbooks/roles/backups/files/ldap_restore.py

@@ -0,0 +1,113 @@
+"""
+Restores LDAP configurations by creating groups, adding users, and associating users with groups using the `cluster` command. Data is loaded from a JSON file at /tmp/ldap_backup.json.
+"""
+import json
+import subprocess
+import progressbar
+import getpass
+
+def run_command(command_list):
+    """Executes a shell command safely and logs the output."""
+    try:
+        subprocess.run(command_list, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to execute command '{' '.join(command_list)}': {e.stderr.decode().strip()}")
+
+def extract_info_from_json(json_file_path):
+    """Extracts group information from the JSON file."""
+    with open(json_file_path, 'r') as file:
+        data = json.load(file)
+    return data.get("groups", {}), data.get("users", {}), data.get("associations", {})
+
+def restore_ldap_configuration(group_info, user_info, user_groups_dict, password):
+    """Executes commands to restore LDAP configurations."""
+    # Define the maximum width of the progress bar
+    MAX_BAR_WIDTH = 50  # Adjust this value as needed
+    MAX_TERMINAL_WIDTH = 80  # Total width of the progress bar display
+
+    # Create groups
+    print("Restoring groups...")
+    bar = progressbar.ProgressBar(
+        max_value=len(group_info),
+        term_width=MAX_TERMINAL_WIDTH,
+        widgets=[
+            progressbar.Bar('=', '[', ']', length=MAX_BAR_WIDTH),
+            ' ',
+            progressbar.Percentage(),
+            ' (', progressbar.SimpleProgress(), ')',
+            ' ', progressbar.Timer(),
+            ' ', progressbar.ETA()
+        ]
+    )
+    for i, (group, gid) in enumerate(group_info.items()):
+        command = ["cluster", "group", "create", group, "--gid", gid]
+        run_command(command)
+        bar.update(i + 1)
+    bar.finish()
+
+    # Add users
+    print("Restoring users...")
+    bar = progressbar.ProgressBar(
+        max_value=len(user_info),
+        term_width=MAX_TERMINAL_WIDTH,
+        widgets=[
+            progressbar.Bar('=', '[', ']', length=MAX_BAR_WIDTH),
+            ' ',
+            progressbar.Percentage(),
+            ' (', progressbar.SimpleProgress(), ')',
+            ' ', progressbar.Timer(),
+            ' ', progressbar.ETA()
+        ]
+    )
+    for i, (user, details) in enumerate(user_info.items()):
+        command = [
+            "cluster", "user", "add", user,
+            "--uid", str(details['uidNumber']),
+            "--gid", str(details['gidNumber']),
+            "--password", password,
+            "--name", details['displayName']
+        ]
+        run_command(command)
+        bar.update(i + 1)
+    bar.finish()
+
+    # Associate users with groups
+    print("Restoring user and group associations...")
+    total_associations = sum(len(groups) for groups in user_groups_dict.values())
+    bar = progressbar.ProgressBar(
+        max_value=total_associations,
+        term_width=MAX_TERMINAL_WIDTH,
+        widgets=[
+            progressbar.Bar('=', '[', ']', length=MAX_BAR_WIDTH),
+            ' ',
+            progressbar.Percentage(),
+            ' (', progressbar.SimpleProgress(), ')',
+            ' ', progressbar.Timer(),
+            ' ', progressbar.ETA()
+        ]
+    )
+    current = 0
+    for user, groups in user_groups_dict.items():
+        for group in groups:
+            command = ["cluster", "group", "add", group, user]
+            run_command(command)
+            current += 1
+            bar.update(current)
+    bar.finish()
+
+def main():
+    path = '/tmp/ldap_backup.json'  # Path to the JSON file
+
+    # Prompt the user for the LDAP password
+    print("Please enter the LDAP user password.")
+    password = getpass.getpass(prompt="Password: ")
+
+    # Extract information from JSON
+    group_info, user_info, association_info = extract_info_from_json(path)
+
+    # Restore LDAP configuration
+    restore_ldap_configuration(group_info, user_info, association_info, password)
+
+# Call the main function to run the program
+if __name__ == '__main__':
+    main()

playbooks/roles/backups/tasks/ldap.yml

@@ -0,0 +1,36 @@
+---
+- name: Install progressbar
+  become: true
+  shell: "/usr/bin/pip install progressbar2"
+
+- name: Install sssd-tools
+  vars: 
+    package_name: 
+      - sssd-tools
+    package_state: latest
+    package_repo: "epel,ol7_developer_EPEL"
+  include_role: 
+    name: safe_yum
+  ignore_errors: true
+
+- name: Copy scripts
+  become: true
+  copy: 
+    src: '{{ item }}'
+    dest: '/opt/oci-hpc/scripts/{{ item }}'
+    force: no
+    owner: '{{ ansible_user }}'
+    group: '{{ ansible_user }}'
+    mode: 0660
+  with_items:
+    - ldap_restore.py
+    - ldap_backup.py
+
+- name: Create crontab entry to backup ldap
+  cron:
+    name: Backup ldap
+    minute: "0"
+    hour: "0"
+    user: '{{ ansible_user }}'
+    job: "python /opt/oci-hpc/scripts/ldap_backup.py"
+    disabled: true

playbooks/roles/backups/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Ensure log directory for backups exists
+  file:
+    path: "/opt/oci-hpc/logs/backups"
+    state: directory
+    owner: '{{ ansible_user }}'
+    group: '{{ ansible_user }}'
+
+- include_tasks: ldap.yml

playbooks/roles/cluster-cli/files/cluster

@@ -31,6 +31,26 @@ def exists(key, value):
     except KeyError:
         return False
 
+def ldap_gid_exists(gid):
+    """ Check if GID exists in LDAP """
+    search_base = groups_dn
+    search_filter = '(gidNumber={})'.format(gid)
+    server = ldap3.Server(host, use_ssl=True)
+    with ldap3.Connection(server, bind_dn, bind_pass, auto_bind=True) as conn:
+        conn.search(search_base, search_filter, attributes=['gidNumber'])
+        return len(conn.entries) > 0
+
+def gid_exists(gid):
+    """ Check if GID exists at the system level or in LDAP """
+    # Check at the system level
+    if exists('gid', gid):
+        return True
+
+    # Check on the LDAP server
+    if ldap_gid_exists(gid):
+        return True
+
+    return False
 
 def find_next_gid(): 
   base_gid = 10005
@@ -187,7 +207,7 @@ def add(user, password, uid, gid, name, nossh):
   if(conn.result['result'] != 0): 
     print(conn.result)
 
-  if exists('gid', gid) is False:
+  if gid_exists(gid) is False:
     click.echo('Creating group')
     conn.add("cn={},{}".format(user, groups_dn), ['top', 'groupOfMembers', 'posixgroup'], { 'cn': [user], 'gidNumber': [gid] } )
     if(conn.result['result'] != 0): 
@@ -196,10 +216,11 @@ def add(user, password, uid, gid, name, nossh):
   homedir='/home/{}/'.format(user)
   os.system("sudo su - "+user+" -c "+" 'ls' 2> /dev/null")
 
-  if not nossh:
+  if nossh is False:
     homedir='/data/{}/'.format(user)
-    os.system("sudo su - "+user+" -c "+"' ssh-keygen -t rsa -b 2048 -q -f "+homedir+".ssh/id_rsa -P \"\"' 2> /dev/null")
-    os.system("sudo su - "+user+" -c "+"'mv "+homedir+".ssh/id_rsa.pub "+homedir+".ssh/authorized_keys' 2> /dev/null")
+    if os.path.exists(homedir) is False:
+      os.system("sudo su - "+user+" -c "+"' ssh-keygen -t rsa -b 2048 -q -f "+homedir+".ssh/id_rsa -P \"\"' 2> /dev/null")
+      os.system("sudo su - "+user+" -c "+"'mv "+homedir+".ssh/id_rsa.pub "+homedir+".ssh/authorized_keys' 2> /dev/null")
 
 @user.command()
 @click.argument('user_name')

playbooks/site.yml

@@ -335,6 +335,10 @@
         name: billing
       when: billing|default(false)|bool
 
+- hosts: controller
+  tasks:
+    - include_role:
+        name: backups
 
 - hosts: compute
   become: true