Skip to content

Commit

Permalink
Rename technology to data #308
Browse files Browse the repository at this point in the history 
The parameter 'technology' is replaced with 'data' wherever it means more than only technologies.
It requires a thorough scan of scripts for a consistent refactoring.
The documentation must be adapted as well.
  • Loading branch information
@deniztepe
deniztepe committed Aug 19, 2022
1 parent ef936f2 commit d2f3ac0
Show file tree
Hide file tree
Showing 10 changed files with 205 additions and 225 deletions.
8 changes: 4 additions & 4 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# bulk download
bulk_date_string = "today"
bulk_cleansing = True
technology_bulk = [
data_bulk = [
"biomass",
"combustion",
"gsgk",
Expand All @@ -43,7 +43,7 @@
api_limit = 10
api_processes = None

technology_api = [
data_api = [
"biomass",
"combustion",
"gsgk",
Expand Down Expand Up @@ -71,7 +71,7 @@
# bulk download
db.download(
method="bulk",
technology=technology_bulk,
data=data_bulk,
bulk_date_string="today",
bulk_cleansing=True,
)
Expand All @@ -83,7 +83,7 @@
api_chunksize=api_chunksize,
api_limit=api_limit,
api_processes=api_processes,
technology=technology_api,
data=data_api,
api_data_types=api_data_types,
api_location_types=api_location_types,
)
Expand Down
54 changes: 26 additions & 28 deletions open_mastr/mastr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,25 @@
from open_mastr.soap_api.mirror import MaStRMirror

from open_mastr.utils.helpers import (
technology_input_harmonisation,
data_input_harmonisation,
print_api_settings,
validate_api_credentials,
)
from open_mastr.utils.config import create_data_dir, get_data_version_dir
from open_mastr.utils.config import (
create_data_dir,
get_data_version_dir,
get_project_home_dir,
)
import open_mastr.utils.orm as orm
from open_mastr.utils.data_io import cleaned_data


# import initialize_database dependencies
from open_mastr.utils.helpers import (
create_database_engine,
validate_parameter_format_for_download_method,
validate_parameter_format_for_mastr_init,
parse_date_string,
)
import open_mastr.utils.orm as orm
from open_mastr.utils.config import get_project_home_dir


class Mastr:
Expand Down Expand Up @@ -65,7 +67,7 @@ def __init__(self, engine="sqlite") -> None:
def download(
self,
method="bulk",
technology=None,
data=None,
bulk_date_string="today",
bulk_cleansing=True,
api_processes=None,
Expand All @@ -86,12 +88,12 @@ def download(
zipped bulk download or via the MaStR API. The latter requires an account
from marktstammdatenregister.de,
(see :ref:`Configuration <Configuration>`). Default to 'bulk'.
technology: str or list or None, optional
Determines which technologies are written to the database. If None, all technologies are
data: str or list or None, optional
Determines which types of data are written to the database. If None, all data is
used. If it is a list, possible entries are "wind", "solar", "biomass", "hydro", "gsgk",
"combustion", "nuclear", "gas", "storage", "electricity_consumer", "location", "market",
"grid", "balancing_area" or "permit". If only one technology is of interest, this can be
given as a string. Default to None, where all technologies are included.
"grid", "balancing_area" or "permit". If only one data is of interest, this can be
given as a string. Default to None, where all data is included.
bulk_date_string: str, optional
Either "today" if the newest data dump should be downloaded from the MaStR website. If
an already downloaded dump should be used, state the date of the download in the format
Expand Down Expand Up @@ -146,7 +148,7 @@ def download(

validate_parameter_format_for_download_method(
method=method,
technology=technology,
data=data,
bulk_date_string=bulk_date_string,
bulk_cleansing=bulk_cleansing,
api_processes=api_processes,
Expand All @@ -172,17 +174,17 @@ def download(
write_mastr_xml_to_database(
engine=self.engine,
zipped_xml_file_path=zipped_xml_file_path,
technology=technology,
data=data,
bulk_cleansing=bulk_cleansing,
bulk_download_date=bulk_download_date,
)

if method == "API":
validate_api_credentials()
if isinstance(technology, str):
technology = [technology]
elif technology is None:
technology = [
if isinstance(data, str):
data = [data]
elif data is None:
data = [
"wind",
"biomass",
"combustion",
Expand All @@ -192,12 +194,8 @@ def download(
"storage",
"solar",
]
(
harm_log,
api_data_types,
api_location_types,
) = technology_input_harmonisation(
technology=technology,
(harm_log, api_data_types, api_location_types,) = data_input_harmonisation(
data=data,
api_data_types=api_data_types,
api_location_types=api_location_types,
)
Expand All @@ -212,7 +210,7 @@ def download(

print_api_settings(
harmonisation_log=harm_log,
technology=technology,
data=data,
api_date=api_date,
api_data_types=api_data_types,
api_chunksize=api_chunksize,
Expand All @@ -227,11 +225,11 @@ def download(
restore_dump=None,
)
# Download basic unit data
mastr_mirror.backfill_basic(technology, limit=api_limit, date=api_date)
mastr_mirror.backfill_basic(data, limit=api_limit, date=api_date)

# Download additional unit data
for tech in technology:
# mastr_mirror.create_additional_data_requests(tech)
for tech in data:
# mastr_mirror.create_additional_data_requests(data)
for data_type in api_data_types:
mastr_mirror.retrieve_additional_data(
tech, data_type, chunksize=api_chunksize, limit=api_limit
Expand Down Expand Up @@ -267,7 +265,7 @@ def to_csv(
chunksize: int
Defines the chunksize of the tables export. Default value is 500.000.
limit: None or int
Limits the number of exported technology and location units.
Limits the number of exported data and location units.
"""

create_data_dir()
Expand Down Expand Up @@ -336,7 +334,7 @@ def to_csv(
if technologies_to_export:
# fill basic unit table, after downloading with method = 'bulk' to use API export functions
api_export.reverse_fill_basic_units(technology=technologies_to_export)
# export to csv per technology
# export to csv per data
api_export.to_csv(
technology=technologies_to_export,
statistic_flag=None,
Expand Down
Loading

0 comments on commit d2f3ac0

Please # to comment.