Skip to content

Commit

Permalink
Set Max Name Length
Browse files Browse the repository at this point in the history
TIL Postgres has a maximum table name length of 63 characters. I set an
upper bound of 58 on the name length of the MetaChangeset. Why 58 and
not 63? Because when we create the materialized view for the data set we
tack on "_view" to the table name.

Updates #235
  • Loading branch information
Vince Forgione committed Aug 21, 2018
1 parent 181f7d4 commit 4704472
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 5 deletions.
14 changes: 9 additions & 5 deletions lib/plenario/actions/data_set_actions.ex
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ defmodule Plenario.Actions.DataSetActions do
@copy "db-actions/etl/copy.sql.eex"
@refresh_view "db-actions/etl/refresh-view.sql.eex"

# ms s m
@etl_timeout 1000 * 60 * 20

def etl!(%Meta{id: meta_id}, download_path), do: etl!(meta_id, download_path)
def etl!(meta_id, download_path) do
meta = MetaActions.get(meta_id)
Expand All @@ -150,13 +153,14 @@ defmodule Plenario.Actions.DataSetActions do
sql_stream = Ecto.Adapters.SQL.stream(Repo, cmd)
file_stream = File.stream!(download_path, [:utf8])

Repo.transaction fn ->
Repo.transaction(fn ->
execute! @truncate_table, table_name: table_name

Repo.transaction(fn -> Enum.into(file_stream, sql_stream) end)

# set the load to infinite time out knowing that the surrounding timeout will
# terminate. this needs to be bumped because the default time out is 2 minutes,
# which for many data sets is far too short to load the entire document.
Repo.transaction(fn -> Enum.into(file_stream, sql_stream) end, timeout: :infinity)
execute! @refresh_view, view_name: view_name
end
end, timeout: @etl_timeout)

:ok
end
Expand Down
3 changes: 3 additions & 0 deletions lib/plenario/changesets/meta_changesets.ex
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ defmodule Plenario.Changesets.MetaChangesets do
|> validate_required(@required_keys)
|> unique_constraint(:source_url)
|> unique_constraint(:name)
|> validate_length(:name, max: 58)
|> cast_assoc(:user)
|> test_source_url()
|> validate_source_type()
Expand All @@ -71,6 +72,7 @@ defmodule Plenario.Changesets.MetaChangesets do
|> validate_required(@required_keys)
|> unique_constraint(:source_url)
|> unique_constraint(:name)
|> validate_length(:name, max: 58)
|> cast_assoc(:user)
|> validate_state()
|> test_source_url()
Expand All @@ -79,6 +81,7 @@ defmodule Plenario.Changesets.MetaChangesets do
|> validate_refresh_interval()
|> validate_refresh_ends_on()
|> set_slug()
|> set_table_name()
end

defp validate_state(%Ecto.Changeset{valid?: true, changes: changes} = changeset) do
Expand Down

0 comments on commit 4704472

Please # to comment.