Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Ok 803 #103

Merged
merged 2 commits into from
Jan 29, 2025
Merged

Ok 803 #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce770f0
DBT:n lokien ja dokumentaation kopiointi S3-ämpäriin
augustk Jan 29, 2025
29ac248
Content-type kopioitaville lokeille
augustk Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions cdk/lib/ecs-stack.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,22 @@ export class EcsStack extends cdk.Stack {
dbtRunnerImageVersion
);

const ovaraDokumentaatioBucket = s3.Bucket.fromBucketName(
this,
`${config.environment}-ovara-dokumentaatio`,
`${config.environment}-ovara-dokumentaatio`
);
const dbtLogsBucket = new s3.Bucket(this, `${config.environment}-dbt-logs`, {
bucketName: `${config.environment}-dbt-logs`,
objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_ENFORCED,
blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
serverAccessLogsBucket: new s3.Bucket(
this,
`${config.environment}-dbt-logs-bucket-server-access-logs`
),
versioned: false,
});

const dbtRunnerSchedule = appscaling.Schedule.cron(config.dbtCron);
const dbtProcessingEnabled = config.dbtProcessingEnabled?.toLowerCase() === 'true';
const dbtRunnerScheduledFargateTask = new ecsPatterns.ScheduledFargateTask(
Expand All @@ -122,6 +138,8 @@ export class EcsStack extends cdk.Stack {
POSTGRES_HOST_PROD: `raportointi.db.${config.publicHostedZone}`,
DBT_PORT_PROD: '5432',
DBT_USERNAME_PROD: 'app',
OVARA_DOC_BUCKET: ovaraDokumentaatioBucket.bucketName,
DBT_LOGS_BUCKET: dbtLogsBucket.bucketName,
},
secrets: {
DBT_PASSWORD_PROD: ecs.Secret.fromSsmParameter(
Expand Down Expand Up @@ -154,6 +172,11 @@ export class EcsStack extends cdk.Stack {
})
);

ovaraDokumentaatioBucket.grantReadWrite(
dbtRunnerScheduledFargateTask.taskDefinition.taskRole
);
dbtLogsBucket.grantReadWrite(dbtRunnerScheduledFargateTask.taskDefinition.taskRole);

dbtRunnerScheduledFargateTask.taskDefinition
.obtainExecutionRole()
.grantAssumeRole(props.githubActionsDeploymentRole);
Expand Down
4 changes: 3 additions & 1 deletion dbt-container/install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ echo $ARCHITECTURE
echo "Installing needed software"
apk --no-cache add \
python3 \
py3-pip
py3-pip \
aws-cli \
tzdata

ln -sf /usr/bin/python3 /usr/bin/python
ln -sf /usr/bin/pip3 /usr/bin/pip
Expand Down
13 changes: 13 additions & 0 deletions dbt-container/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,17 @@ start=$(date +%s)
dbt run-operation tempdata_cleanup --target=prod
echo "Siivouksen kesto `expr $(date +%s) - ${start}` s"

echo "Generoidaan dokumentaatio"
dbt docs generate --target=prod

echo "Kopioidaan dokumentaatio S3:een"
aws s3 cp ./target/catalog.json s3://$OVARA_DOC_BUCKET/dbt/catalog.json
aws s3 cp ./target/index.html s3://$OVARA_DOC_BUCKET/dbt/index.html
aws s3 cp ./target/manifest.json s3://$OVARA_DOC_BUCKET/dbt/manifest.json

echo "Kopioidaan lokit S3:een"
CURRENT_TIME="$(TZ=Europe/Helsinki date +%Y-%m-%d_%H:%M:%S%Z)"
echo "$CURRENT_TIME"
aws s3 cp ./logs s3://$DBT_LOGS_BUCKET/$CURRENT_TIME --recursive --include 'logs/dbt.log*' --content-type 'text/plain'

exit 0
Loading