Finished refactoring for object-oriented

hedweb/columns.py

@@ -1,4 +1,3 @@
-from flask import current_app
 import openpyxl
 import os
 
@@ -10,9 +9,6 @@
 from hedweb.web_util import form_has_file, form_has_option
 
 
-app_config = current_app.config
-
-
 def create_column_selections(form_dict):
     """ Return a tag prefix dictionary from a form dictionary.
 
@@ -62,7 +58,7 @@ def _create_columns_info(columns_file, has_column_names: True, sheet_name: None)
     Args:
         columns_file (File-like): File to create the dictionary for.
         has_column_names (bool):  If True, first row is interpreted as the column names.
-        sheet_name (str): The name of the worksheet if this is an excel file.
+        sheet_name (str): The name of the worksheet if this is an Excel file.
 
     Returns:
         dict: Dictionary containing information include column names and number of unique values in each column.
@@ -141,33 +137,27 @@ def get_columns_request(request):
     return _create_columns_info(columns_file, has_column_names, sheet_name)
 
 
-def get_prefix_dict(form_dict):
+def get_column_names(form_dict):
     """ Return a tag prefix dictionary from a form dictionary.
 
-    Args:
-        form_dict (dict): The dictionary returned from a form that contains a column prefix table.
+    Parameters:
+        form_dict (dict): The dictionary returned from a form that contains a column table.
 
     Returns:
         list: List of selected columns
-        dict: Prefix dictionary
-
-    Note: The form counts columns starting from 1 but prefix dictionary starts with index 0.
+        
+    Note: The form counts columns starting from 1.
     """
     tag_columns = []
-    prefix_dict = {}
     keys = form_dict.keys()
     for key in keys:
         index_check = key.rfind('_check')
         if index_check == -1 or form_dict[key] != 'on':
             continue
         pieces = key.split("_")
         column_number = int(pieces[1])
-        info_key = key[0: index_check] + "_input"
-        if form_dict.get(info_key, None):
-            prefix_dict[column_number] = form_dict[info_key]
-        else:
-            tag_columns.append(column_number)
-    return tag_columns, prefix_dict
+        tag_columns.append(column_number)
+    return tag_columns
 
 
 def _get_worksheet(excel_file, sheet_name):

hedweb/process_events.py

@@ -11,12 +11,13 @@
 from hed.models.df_util import get_assembled, shrink_defs
 from hed.tools.util.data_util import separate_values
 from hed.tools.remodeling.dispatcher import Dispatcher
+from hed.tools.remodeling.validator import RemodelerValidator
 from hed.tools.analysis import analysis_util
 from hed.tools.analysis.tabular_summary import TabularSummary
 from hed.tools.analysis.annotation_util import generate_sidecar_entry
 from hedweb.constants import base_constants
 from hedweb.process_base import ProcessBase
-from hedweb.columns import create_column_selections, create_columns_included
+from hedweb.columns import create_column_selections
 from hedweb.web_util import form_has_option, generate_filename, get_hed_schema_from_pull_down, get_schema_versions
 
 class ProcessEvents(ProcessBase):
@@ -68,7 +69,6 @@ def set_input_from_form(self, request):
         self.expand_defs = form_has_option(request, base_constants.EXPAND_DEFS, 'on')
         self.include_summaries = form_has_option(request, base_constants.INCLUDE_SUMMARIES, 'on')
         self.columns_selected = create_column_selections(request.form)
-        self.columns_included = create_columns_included(request.form)
         if self.command == base_constants.COMMAND_ASSEMBLE:
             self.columns_included = ['onset']   # TODO  add user interface option to choose columns.
         if self.command != base_constants.COMMAND_GENERATE_SIDECAR:
@@ -206,9 +206,10 @@ def remodel(self):
         display_name = self.events.name
         remodel_name = self.remodel_operations['name']
         operations = self.remodel_operations['operations']
-        operations_list, errors = Dispatcher.parse_operations(operations)
+        validator = RemodelerValidator()
+        errors = validator.validate(operations)
         if errors:
-            issue_str = Dispatcher.errors_to_str(errors)
+            issue_str = "\n".join(errors)
             file_name = generate_filename(remodel_name, name_suffix='_operation_parse_errors',
                                           extension='.txt', append_datetime=True)
             return {base_constants.COMMAND: base_constants.COMMAND_REMODEL,

hedweb/process_schemas.py

@@ -27,7 +27,7 @@ def set_input_from_dict(self, input_dict):
         """
         self.schema = input_dict.get(base_constants.SCHEMA, None)
         self.schema2 = input_dict.get(base_constants.SCHEMA2, None)
-        self.command = input_dict.get(base_constants.COMMAND_OPTION, '')
+        self.command = input_dict.get(base_constants.COMMAND, '')
         self.check_for_warnings = input_dict.get(base_constants.CHECK_FOR_WARNINGS, False)
 
     def set_input_from_form(self, request):

hedweb/process_services.py

@@ -8,8 +8,8 @@
 from hed.models.tabular_input import TabularInput
 from hed.errors import HedFileError
 from hed import schema as hedschema
-from hedweb.columns import get_prefix_dict
 from hedweb.constants import base_constants
+from hedweb.columns import get_column_names
 from hedweb.process_events import ProcessEvents
 from hedweb.process_schemas import ProcessSchemas
 from hedweb.process_spreadsheets import ProcessSpreadsheets
@@ -73,6 +73,8 @@ def set_column_parameters(arguments, params):
             for column in params['columns_included']:
                 columns_included.append(column)
         arguments[base_constants.COLUMNS_INCLUDED] = columns_included
+        arguments[base_constants.TAG_COLUMNS] = get_column_names(params)
+        arguments[base_constants.HAS_COLUMN_NAMES] = True
 
     @staticmethod
     def set_sidecar(arguments, params):
@@ -116,12 +118,10 @@ def set_input_objects(arguments, params):
                 TabularInput(file=io.StringIO(params[base_constants.EVENTS_STRING]),
                              sidecar=arguments.get(base_constants.SIDECAR, None), name='Events')
         if base_constants.SPREADSHEET_STRING in params and params[base_constants.SPREADSHEET_STRING]:
-            tag_columns, prefix_dict = get_prefix_dict(params)
-            has_column_names = arguments.get(base_constants.HAS_COLUMN_NAMES, None)
             arguments[base_constants.SPREADSHEET] = \
                 SpreadsheetInput(file=io.StringIO(params[base_constants.SPREADSHEET_STRING]), file_type=".tsv",
-                                 tag_columns=tag_columns, has_column_names=has_column_names,
-                                 column_prefix_dictionary=prefix_dict, name='spreadsheets.tsv')
+                                 tag_columns=arguments[base_constants.TAG_COLUMNS], 
+                                 has_column_names=True, column_prefix_dictionary=None, name='spreadsheets.tsv')
         if base_constants.STRING_LIST in params and params[base_constants.STRING_LIST]:
             s_list = []
             for s in params[base_constants.STRING_LIST]:
@@ -162,7 +162,7 @@ def get_service_info(params):
         if command != "get_services" and len(pieces) == 2:
             command = pieces[1]
             command_target = pieces[0]
-        has_column_names = params.get(base_constants.HAS_COLUMN_NAMES, '') == 'on'
+        has_column_names = True
         expand_defs = params.get(base_constants.EXPAND_DEFS, '') == 'on'
         check_for_warnings = params.get(base_constants.CHECK_FOR_WARNINGS, '') == 'on'
         include_description_tags = params.get(base_constants.INCLUDE_DESCRIPTION_TAGS, '') == 'on'
@@ -174,8 +174,6 @@ def get_service_info(params):
                 base_constants.CHECK_FOR_WARNINGS: check_for_warnings,
                 base_constants.EXPAND_DEFS: expand_defs,
                 base_constants.INCLUDE_DESCRIPTION_TAGS: include_description_tags
-                # base_constants.TAG_COLUMNS: tag_columns,
-                # base_constants.COLUMN_PREFIX_DICTIONARY: prefix_dict
                 }
 
     @staticmethod
@@ -229,6 +227,7 @@ def process(arguments):
             response["results"] = proc_obj.process()
         results = response.get("results", {})
         results["software_version"] = app_config['VERSIONS']
+        results = ProcessServices.package_spreadsheet(results)
         response["results"] = results
         return response
 

hedweb/process_spreadsheets.py

@@ -5,15 +5,12 @@
 from hed.errors import get_printable_issue_string, HedFileError, ErrorHandler
 from hed.models.spreadsheet_input import SpreadsheetInput
 from hedweb.web_util import get_schema_versions
-
 from hedweb.constants import base_constants, file_constants
-from hedweb.columns import get_prefix_dict
 from hedweb.process_base import ProcessBase
+from hedweb.columns import get_column_names
 from hedweb.web_util import filter_issues, form_has_option, generate_filename, get_hed_schema_from_pull_down
 
 
-app_config = current_app.config
-
 class ProcessSpreadsheets(ProcessBase):
 
     def __init__(self):
@@ -43,9 +40,8 @@ def set_input_from_dict(self, input_dict):
         self.spreadsheet = input_dict.get(base_constants.SPREADSHEET, None)
         self.worksheet = input_dict.get(base_constants.WORKSHEET, None)
         self.spreadsheet_type = input_dict.get(base_constants.SPREADSHEET_TYPE, file_constants.TSV_EXTENSION)
-        self.prefix_dict = input_dict.get(base_constants.COLUMN_PREFIX_DICTIONARY, {})
         self.tag_columns = input_dict.get(base_constants.TAG_COLUMNS, [])
-        self.has_column_names = input_dict.get(base_constants.HAS_COLUMN_NAMES, True)
+        self.has_column_names = True
         self.check_for_warnings = input_dict.get(base_constants.CHECK_FOR_WARNINGS, False)
         self.expand_defs = input_dict.get(base_constants.EXPAND_DEFS, False)
 
@@ -59,11 +55,9 @@ def set_input_from_form(self, request):
         self.schema = get_hed_schema_from_pull_down(request)
         self.worksheet = request.form.get(base_constants.WORKSHEET_NAME, None)
         self.command = request.form.get(base_constants.COMMAND_OPTION, '')
-        self.has_column_names =  form_has_option(request, base_constants.HAS_COLUMN_NAMES, 'on')
+        self.has_column_names =  True
         self.check_for_warnings = form_has_option(request, base_constants.CHECK_FOR_WARNINGS, 'on')
-        self.tag_columns, self.prefix_dict = get_prefix_dict(request.form)
-        if self.command != base_constants.COMMAND_VALIDATE:
-            self.prefix_dict = {}
+        self.tag_columns = get_column_names(request.form)
         filename = request.files[base_constants.SPREADSHEET_FILE].filename
         file_ext = os.path.splitext(filename)[1]
         if file_ext in file_constants.EXCEL_FILE_EXTENSIONS:
@@ -73,7 +67,7 @@ def set_input_from_form(self, request):
                                             worksheet_name=self.worksheet,
                                             tag_columns=self.tag_columns,
                                             has_column_names=self.has_column_names,
-                                            column_prefix_dictionary=self.prefix_dict,
+                                            column_prefix_dictionary=None,
                                             name=filename)
 
     def process(self):

hedweb/process_strings.py

@@ -30,7 +30,7 @@ def set_input_from_dict(self, input_dict):
             dict: A dictionary of schema processing parameters in standard form.
 
         """
-        self.command = input_dict.get(base_constants.COMMAND_OPTION, '')
+        self.command = input_dict.get(base_constants.COMMAND, '')
         self.schema = input_dict.get(base_constants.SCHEMA, None)
         self.string_list = input_dict.get(base_constants.STRING_LIST, '')
         self.definitions = input_dict.get(base_constants.DEFINITIONS, None)

hedweb/routes.py

@@ -133,7 +133,7 @@ def services_results():
     try:
         hedschema.set_cache_directory(current_app.config['HED_CACHE_FOLDER'])
         hedschema.cache_xml_versions()
-        arguments = ProcessServices.get_input_from_request(request)
+        arguments = ProcessServices.set_input_from_request(request)
         response = ProcessServices.process(arguments)
         return json.dumps(response)
     except Exception as ex:

hedweb/static/resources/services.json

@@ -142,8 +142,7 @@
                     "schema_version"
                 ],
                 [
-                    "column_x_input",
-                    "column_x_check"
+                    "column_x_input"
                 ],
                 "check_for_warnings"
             ],
@@ -159,8 +158,7 @@
                     "schema_version"
                 ],
                 [
-                    "column_x_input",
-                    "column_x_check"
+                    "column_x_input"
                 ]
             ],
             "Returns": "The converted spreadsheet or a list of issues if any."
@@ -175,8 +173,7 @@
                     "schema_version"
                 ],
                 [
-                    "column_x_input",
-                    "column_x_check"
+                    "column_x_input"
                 ]
             ],
             "Returns": "The converted spreadsheet or a list of issues if any."
@@ -222,7 +219,6 @@
     "parameter_meanings": {
         "check_for_warnings": "If true, check for warnings when processing HED strings.",
         "column_x_check": "If present with value 'on' column x has HED tags. (x is column position starting at 0.)",
-        "column_x_input": "The prefix prepended to column number x (starting at 0) if column x has HED tags.",
         "columns_categorical": "A list of names of categorical event file columns",
         "columns_included": "A list of names of columns to be included for assembly or query.",
         "columns_value": "A list of names of value event file columns",

hedweb/templates/js/column-info.js

@@ -107,7 +107,7 @@ function showEvents(columnList, columnCounts) {
  * @param {boolean} hasPrefixes - if true then the prefix inputs are displayed.
  * 
  */
-function showIndices(columnList, hasPrefixes=true) {
+function showIndices(columnList, hasPrefixes=false) {
     $('#show_indices_section').show();
     let indicesTable = $('#show_indices_table');
     let contents = '<thead><tr><th scope="col">Include?</th><th scope="col">Column names</th>';

hedweb/templates/js/spreadsheets-form.js

@@ -37,7 +37,6 @@ $('#spreadsheet_clear').on('click', function () {
  */
 function clearForm() {
     clearFlashMessages();
-
     clearSpreadsheet()
     $("#validate").prop('checked', true);
     setOptions();

hedweb/templates/js/spreadsheets-input.js

@@ -92,6 +92,6 @@ function setIndicesTable() {
         let info = getColumnsInfo(spreadsheetFile, 'spreadsheet_flash', worksheet, true)
         let cols = info['column_list']
         let selectedElement = document.getElementById("process_actions");
-        showIndices(cols, selectedElement.value === "validate")
+        showIndices(cols, false)
     }
 }

hedweb/web_util.py

@@ -4,7 +4,7 @@
 import os
 import zipfile
 from urllib.parse import urlparse
-from flask import current_app, Response, make_response, send_file
+from flask import Response, make_response, send_file
 from werkzeug.utils import secure_filename
 
 from hed import schema as hedschema
@@ -13,8 +13,6 @@
 from hed.errors import HedFileError, ErrorSeverity, ErrorHandler
 from hedweb.constants import base_constants, file_constants
 
-app_config = current_app.config
-
 TIME_FORMAT = '%Y_%m_%d_T_%H_%M_%S_%f'
 
 
@@ -171,10 +169,10 @@ def generate_filename(base_name, name_prefix=None, name_suffix=None, extension=N
     """ Generate a filename for the attachment.
 
     Parameters:
-        base_name (str):   Name of the base, usually the name of the file that the issues were generated from.
-        name_prefix (str): Prefix prepended to the front of the base name.
-        name_suffix (str): Suffix appended to the end of the base name.
-        extension (str):   Extension to use.
+        base_name (str or None):   Name of the base, usually the name of the file that the issues were generated from.
+        name_prefix (str or None): Prefix prepended to the front of the base name.
+        name_suffix (str or None): Suffix appended to the end of the base name.
+        extension (str or None):   Extension to use.
         append_datetime (bool): If True, append the current date-time to the base output filename.
 
     Returns:

requirements.txt

@@ -2,3 +2,4 @@ flask==2.3.2
 flask-wtf==1.1.1
 config==0.5.1
 Werkzeug==2.3.4
+