From 93de978bda94b17f2eb137e69d66b9920de9433f Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Wed, 28 Aug 2024 16:08:20 +0300 Subject: [PATCH 01/25] feat: simple app slo-hibernate --- .../src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java b/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java index 4e63f14..89c8b1a 100644 --- a/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java +++ b/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java @@ -85,8 +85,8 @@ protected String columnType(int sqlTypeCode) { case SMALLINT -> "Int16"; case INTEGER -> "Int32"; case BIGINT -> "Int64"; - case REAL, FLOAT -> "Float"; - case DOUBLE -> "Double"; + case REAL -> "Float"; + case DOUBLE, FLOAT -> "Double"; case NUMERIC, DECIMAL -> "Decimal (22,9)"; // Fixed case DATE -> "Date"; case JDBC_TYPE_DATETIME_CODE -> "Datetime"; From c1ddc5384ba827381646ab6f2222b840f0ecdf09 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 12:02:46 +0300 Subject: [PATCH 02/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 70 ++ .../provisioning/dashboards/dashboard.yml | 6 + .../grafana/provisioning/dashboards/slo.json | 646 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 11 + slo/config/prometheus/prometheus.yml | 8 + slo/docker-compose.yml | 48 ++ slo/hibernate/Dockerfile | 11 + slo/hibernate/pom.xml | 65 ++ .../tech/hibernate/HibernateApplication.kt | 75 ++ .../ydb/tech/hibernate/entity/SloEntity.kt | 31 + .../repository/SloEntityRepository.kt | 10 + .../ydb/tech/hibernate/retry/YdbRetryable.kt | 19 + .../ydb/tech/hibernate/service/SloService.kt | 37 + .../src/main/resources/application.properties | 5 + .../resources/migration/V1__create_table.sql | 8 + slo/pom.xml | 85 +++ 16 files changed, 1135 insertions(+) create mode 100644 .github/workflows/slo.yaml create mode 100644 slo/config/grafana/provisioning/dashboards/dashboard.yml create mode 100644 slo/config/grafana/provisioning/dashboards/slo.json create mode 100644 slo/config/grafana/provisioning/datasources/datasource.yml create mode 100644 slo/config/prometheus/prometheus.yml create mode 100644 slo/docker-compose.yml create mode 100644 slo/hibernate/Dockerfile create mode 100644 slo/hibernate/pom.xml create mode 100644 slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt create mode 100644 slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt create mode 100644 slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt create mode 100644 slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt create mode 100644 slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt create mode 100644 slo/hibernate/src/main/resources/application.properties create mode 100644 slo/hibernate/src/main/resources/migration/V1__create_table.sql create mode 100644 slo/pom.xml diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml new file mode 100644 index 0000000..d0c5147 --- /dev/null +++ b/.github/workflows/slo.yaml @@ -0,0 +1,70 @@ +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +name: SLO + +jobs: + test-slo: + concurrency: + group: slo-${{ github.ref }} + if: (!contains(github.event.pull_request.labels.*.name, 'no slo')) + + runs-on: ubuntu-latest + name: SLO test + permissions: + checks: write + pull-requests: write + contents: read + issues: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + if: env.DOCKER_REPO != null + env: + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + + - name: Run SLO + uses: ydb-platform/slo-tests@main + if: env.DOCKER_REPO != null + env: + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + continue-on-error: true + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + KUBECONFIG_B64: ${{ secrets.SLO_KUBE_CONFIG }} + AWS_CREDENTIALS_B64: ${{ secrets.SLO_AWS_CREDENTIALS }} + AWS_CONFIG_B64: ${{ secrets.SLO_AWS_CONFIG }} + DOCKER_USERNAME: ${{ secrets.SLO_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SLO_DOCKER_PASSWORD }} + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + DOCKER_FOLDER: ${{ secrets.SLO_DOCKER_FOLDER }} + s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} + s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} + grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} + # grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }} + grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 + grafana_dashboard_width: 2000 + grafana_dashboard_height: 2300 + ydb_version: 'newest' + timeBetweenPhases: 30 + shutdownTime: 30 + + language_id0: 'hibernate' + workload_path0: 'slo/hibernate' + language0: 'Hibernate YQL Dialect over YDB JDBC Driver' + workload_build_context0: ../.. + workload_build_options0: -f Dockerfile + + + - uses: actions/upload-artifact@v3 + if: always() && env.DOCKER_REPO != null + env: + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + with: + name: slo-logs + path: logs/ diff --git a/slo/config/grafana/provisioning/dashboards/dashboard.yml b/slo/config/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..c678414 --- /dev/null +++ b/slo/config/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,6 @@ +apiVersion: 1 + +providers: + - name: 'SLO' + options: + path: /etc/grafana/provisioning/dashboards diff --git a/slo/config/grafana/provisioning/dashboards/slo.json b/slo/config/grafana/provisioning/dashboards/slo.json new file mode 100644 index 0000000..69d76bf --- /dev/null +++ b/slo/config/grafana/provisioning/dashboards/slo.json @@ -0,0 +1,646 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "rate(oks[$__rate_interval]) > 0", + "hide": false, + "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} OK", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "rate(not_oks[$__rate_interval]) > 0", + "hide": false, + "legendFormat": "({{sdk}}-{{sdkVersion}}) {{jobName}} not OK", + "range": true, + "refId": "C" + } + ], + "title": "SLO Requests RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "histogram_quantile(1, rate(attempts_bucket[$__rate_interval]))", + "hide": false, + "legendFormat": "{{sdk}}-{{sdkVersion}} {{jobName}}-{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Attempts", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 7, + "panels": [], + "title": "Latencies", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"read\", status=\"ok\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Read Latencies (OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"write\", status=\"ok\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Latencies (OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"read\", status=\"err\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Read Latencies (NOT OK)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "builder", + "expr": "latency{jobName=\"write\", status=\"err\"} > 0", + "legendFormat": "{{sdk}}-{{sdkVersion}}-p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Latencies (NOT OK)", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "filters": [], + "hide": 0, + "label": "", + "name": "filter", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "SLO", + "uid": "7CzMl5t4k", + "version": 1, + "weekStart": "" +} diff --git a/slo/config/grafana/provisioning/datasources/datasource.yml b/slo/config/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..0b62b9c --- /dev/null +++ b/slo/config/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true diff --git a/slo/config/prometheus/prometheus.yml b/slo/config/prometheus/prometheus.yml new file mode 100644 index 0000000..281b390 --- /dev/null +++ b/slo/config/prometheus/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: 'slo' + static_configs: + - targets: ['prometheus-pushgateway:9091'] diff --git a/slo/docker-compose.yml b/slo/docker-compose.yml new file mode 100644 index 0000000..29f465b --- /dev/null +++ b/slo/docker-compose.yml @@ -0,0 +1,48 @@ +version: '2.1' + +networks: + monitor-net: + driver: bridge + +services: + prometheus: + image: prom/prometheus:v2.44.0 + container_name: prometheus + user: "$UID:$GID" + volumes: + - ./configs/prometheus:/etc/prometheus + - ../data/prometheus:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + ports: + - "9090:9090" + networks: + - monitor-net + + prometheus-pushgateway: + image: prom/pushgateway:v1.6.0 + container_name: prometheus-pushgateway + ports: + - "9091:9091" + networks: + - monitor-net + + grafana: + image: grafana/grafana:9.5.3 + container_name: grafana + user: "$UID:$GID" + volumes: + - ./configs/grafana/provisioning:/etc/grafana/provisioning + - ../data/grafana:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=password + restart: unless-stopped + ports: + - "3000:3000" + networks: + - monitor-net diff --git a/slo/hibernate/Dockerfile b/slo/hibernate/Dockerfile new file mode 100644 index 0000000..3a2e0e0 --- /dev/null +++ b/slo/hibernate/Dockerfile @@ -0,0 +1,11 @@ +FROM maven:3.8.4-openjdk-17 AS build +WORKDIR /app +COPY pom.xml . +RUN mvn dependency:resolve +COPY src ./src +RUN mvn package -DskipTests + +FROM openjdk:17 AS run +WORKDIR /app +COPY --from=build /app/target/hibernate-0.0.1-SNAPSHOT.jar /app/app.jar +ENTRYPOINT ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/slo/hibernate/pom.xml b/slo/hibernate/pom.xml new file mode 100644 index 0000000..e995e78 --- /dev/null +++ b/slo/hibernate/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + ydb.tech + slo + 0.0.1-SNAPSHOT + + hibernate + 0.0.1-SNAPSHOT + hibernate + hibernate + + 21 + 1.9.25 + 0.9.5 + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.retry + spring-retry + + + tech.ydb.dialects + hibernate-ydb-dialect + ${hibernate.ydb.dialect.version} + + + + + ${project.basedir}/src/main/kotlin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.jetbrains.kotlin + kotlin-maven-plugin + + + -Xjsr305=strict + + + spring + jpa + + + + + org.jetbrains.kotlin + kotlin-maven-allopen + ${kotlin.version} + + + + + + + diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt new file mode 100644 index 0000000..2498fcc --- /dev/null +++ b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt @@ -0,0 +1,75 @@ +package tech.ydb.slo.hibernate + +import io.prometheus.metrics.core.metrics.Counter +import io.prometheus.metrics.exporter.pushgateway.PushGateway +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.CommandLineRunner +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication +import tech.ydb.slo.hibernate.service.SloService +import java.time.Duration +import java.time.Instant +import java.util.concurrent.Executors + + +@SpringBootApplication +class HibernateApplication : CommandLineRunner { + + @Autowired + private lateinit var sloService: SloService + + override fun run(vararg args: String) { + if (args.isNotEmpty() && args[0] == "create") { + return // Flyway created slo table + } + + val pushGateway = PushGateway.builder() + .job("hibernate-slo") + .address("prometheus-pushgateway:9091") + .build() + + val okCount = Counter.builder().name("oks") + .labelNames("jobName", "sdk", "sdkVersion") + .help("Count of OK") + .register() + val notOkCount = Counter.builder().name("not_oks") + .labelNames("jobName", "sdk", "sdkVersion") + .help("Count of not OK") + .register() + + val workersRead = Executors.newFixedThreadPool(3) + val workersWrite = Executors.newFixedThreadPool(1) + + val startTime = Instant.now() + + while (Instant.now() < startTime.plus(Duration.ofSeconds(10))) { + workersRead.submit { + try { + sloService.find() + okCount.labelValues("read", "TODO", "hibernate").inc() + } catch (e: Exception) { + notOkCount.labelValues("read", "TODO", "hibernate").inc() + println(e) + } + pushGateway.push() + } + + workersWrite.submit { + try { + sloService.save() + okCount.labelValues("write", "TODO", "hibernate").inc() + } catch (e: Exception) { + notOkCount.labelValues("write", "TODO", "hibernate").inc() + println(e) + } + pushGateway.push() + } + } + + pushGateway.delete() + } +} + +fun main(args: Array) { + runApplication(*args) +} diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt new file mode 100644 index 0000000..43a4ab6 --- /dev/null +++ b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt @@ -0,0 +1,31 @@ +package tech.ydb.slo.hibernate.entity + +import jakarta.annotation.Nonnull +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.Instant + +/** + * @author Kirill Kurdyukov + */ +@Entity +@Table(name = "slo-hibernate") +class SloEntity { + + @Id + var id: Int = 0 + + @Nonnull + @Column(name = "payload_str") + lateinit var payloadStr: String + + @Nonnull + @Column(name = "payload_double") + var payloadDouble: Double = 0.0 + + @Nonnull + @Column(name = "payload_timestamp") + lateinit var payloadTimestamp: Instant +} \ No newline at end of file diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt new file mode 100644 index 0000000..adf3a2b --- /dev/null +++ b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt @@ -0,0 +1,10 @@ +package tech.ydb.slo.hibernate.repository + +import org.springframework.data.repository.CrudRepository +import tech.ydb.slo.hibernate.entity.SloEntity + + +/** + * @author Kirill Kurdyukov + */ +interface SloEntityRepository : CrudRepository \ No newline at end of file diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt new file mode 100644 index 0000000..ff31cad --- /dev/null +++ b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt @@ -0,0 +1,19 @@ +package tech.ydb.slo.hibernate.retry + +import org.springframework.retry.annotation.Backoff +import org.springframework.retry.annotation.Retryable +import java.sql.SQLRecoverableException + +@Target( + AnnotationTarget.FUNCTION, + AnnotationTarget.PROPERTY_GETTER, + AnnotationTarget.PROPERTY_SETTER, + AnnotationTarget.CLASS +) +@Retention(AnnotationRetention.RUNTIME) +@Retryable( + retryFor = [SQLRecoverableException::class], + maxAttempts = 5, + backoff = Backoff(delay = 100, multiplier = 2.0, maxDelay = 5000, random = true), +) +annotation class YdbRetryable \ No newline at end of file diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt new file mode 100644 index 0000000..b49bd3d --- /dev/null +++ b/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt @@ -0,0 +1,37 @@ +package tech.ydb.slo.hibernate.service + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import tech.ydb.slo.hibernate.entity.SloEntity +import tech.ydb.slo.hibernate.repository.SloEntityRepository +import tech.ydb.slo.hibernate.retry.YdbRetryable +import java.time.Instant +import java.util.Optional +import java.util.UUID +import java.util.concurrent.ThreadLocalRandom +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author Kirill Kurdyukov + */ +@Service +class SloService { + private val id = AtomicInteger(1) + + @Autowired + private lateinit var sloEntityRepository: SloEntityRepository + + @YdbRetryable + fun save() = sloEntityRepository.save( + SloEntity().apply { + id = this@SloService.id.getAndIncrement() + payloadStr = UUID.randomUUID().toString() + payloadDouble = ThreadLocalRandom.current().nextDouble() + payloadTimestamp = Instant.now() + } + ) + + @YdbRetryable + fun find(): Optional = + sloEntityRepository.findById(ThreadLocalRandom.current().nextInt(id.get())) +} \ No newline at end of file diff --git a/slo/hibernate/src/main/resources/application.properties b/slo/hibernate/src/main/resources/application.properties new file mode 100644 index 0000000..24114da --- /dev/null +++ b/slo/hibernate/src/main/resources/application.properties @@ -0,0 +1,5 @@ +spring.application.name=hibernate +spring.datasource.url=jdbc:ydb:grpc://database-sample-grpc:2135/root/database-sample +spring.jpa.properties.hibernate.dialect=tech.ydb.hibernate.dialect.YdbDialect + +spring.datasource.driver-class-name=tech.ydb.jdbc.YdbDriver diff --git a/slo/hibernate/src/main/resources/migration/V1__create_table.sql b/slo/hibernate/src/main/resources/migration/V1__create_table.sql new file mode 100644 index 0000000..50f4960 --- /dev/null +++ b/slo/hibernate/src/main/resources/migration/V1__create_table.sql @@ -0,0 +1,8 @@ +CREATE TABLE `slo-hibernate` +( + id Int32, + payload_str Text, + payload_double Double, + payload_timestamp Timestamp, + PRIMARY KEY (id) +) \ No newline at end of file diff --git a/slo/pom.xml b/slo/pom.xml new file mode 100644 index 0000000..fc678f9 --- /dev/null +++ b/slo/pom.xml @@ -0,0 +1,85 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.3.3 + + + ydb.tech + slo + 0.0.1-SNAPSHOT + slo + pom + slo + + 21 + 1.9.25 + 2.2.3 + 1.3.1 + + + hibernate + + + + org.jetbrains.kotlin + kotlin-reflect + + + org.jetbrains.kotlin + kotlin-stdlib + + + org.flywaydb + flyway-core + + + tech.ydb.dialects + flyway-ydb-dialect + 1.0.0-RC0 + + + tech.ydb.jdbc + ydb-jdbc-driver + ${ydb.jdbc.driver.version} + + + + + io.prometheus + prometheus-metrics-core + ${prometheus.version} + + + io.prometheus + prometheus-metrics-config + ${prometheus.version} + + + io.prometheus + prometheus-metrics-exporter-pushgateway + ${prometheus.version} + + + + + + + org.jetbrains.kotlin + kotlin-maven-plugin + + + compile + compile + + compile + + + + + + + From fe7d839cf61cbdebc1ec3cd730b47290a7008062 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 12:21:37 +0300 Subject: [PATCH 03/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 4 ++-- .../src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index d0c5147..d4ccc80 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -1,8 +1,8 @@ on: push: - branches: [main] + branches: [ main ] pull_request: - branches: [main] + branches: [ main ] workflow_dispatch: name: SLO diff --git a/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java b/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java index 89c8b1a..4e63f14 100644 --- a/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java +++ b/hibernate-dialect/src/main/java/tech/ydb/hibernate/dialect/YdbDialect.java @@ -85,8 +85,8 @@ protected String columnType(int sqlTypeCode) { case SMALLINT -> "Int16"; case INTEGER -> "Int32"; case BIGINT -> "Int64"; - case REAL -> "Float"; - case DOUBLE, FLOAT -> "Double"; + case REAL, FLOAT -> "Float"; + case DOUBLE -> "Double"; case NUMERIC, DECIMAL -> "Decimal (22,9)"; // Fixed case DATE -> "Date"; case JDBC_TYPE_DATETIME_CODE -> "Datetime"; From 4527878fee78946721f7eceb7b0760cff82f1154 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 13:45:46 +0300 Subject: [PATCH 04/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index d4ccc80..b7d023a 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -29,7 +29,7 @@ jobs: DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} - name: Run SLO - uses: ydb-platform/slo-tests@main + uses: ydb-platform/slo-tests@k8s-local if: env.DOCKER_REPO != null env: DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} From 04d59307e5f96b1f1372b2a81e191f66707e4684 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 13:48:12 +0300 Subject: [PATCH 05/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index b7d023a..eca120b 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -30,29 +30,29 @@ jobs: - name: Run SLO uses: ydb-platform/slo-tests@k8s-local - if: env.DOCKER_REPO != null - env: - DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} +# if: env.DOCKER_REPO != null +# env: +# DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} continue-on-error: true with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - KUBECONFIG_B64: ${{ secrets.SLO_KUBE_CONFIG }} - AWS_CREDENTIALS_B64: ${{ secrets.SLO_AWS_CREDENTIALS }} - AWS_CONFIG_B64: ${{ secrets.SLO_AWS_CONFIG }} - DOCKER_USERNAME: ${{ secrets.SLO_DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.SLO_DOCKER_PASSWORD }} - DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} - DOCKER_FOLDER: ${{ secrets.SLO_DOCKER_FOLDER }} - s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} - s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} - grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# KUBECONFIG_B64: ${{ secrets.SLO_KUBE_CONFIG }} +# AWS_CREDENTIALS_B64: ${{ secrets.SLO_AWS_CREDENTIALS }} +# AWS_CONFIG_B64: ${{ secrets.SLO_AWS_CONFIG }} +# DOCKER_USERNAME: ${{ secrets.SLO_DOCKER_USERNAME }} +# DOCKER_PASSWORD: ${{ secrets.SLO_DOCKER_PASSWORD }} +# DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} +# DOCKER_FOLDER: ${{ secrets.SLO_DOCKER_FOLDER }} +# s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} +# s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} +# grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} # grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }} - grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 - grafana_dashboard_width: 2000 - grafana_dashboard_height: 2300 - ydb_version: 'newest' - timeBetweenPhases: 30 - shutdownTime: 30 +# grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 +# grafana_dashboard_width: 2000 +# grafana_dashboard_height: 2300 +# ydb_version: 'newest' +# timeBetweenPhases: 30 +# shutdownTime: 30 language_id0: 'hibernate' workload_path0: 'slo/hibernate' From 4ac7f9936253286e88bd4e1ca0a36f921bb4d37c Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 14:10:05 +0300 Subject: [PATCH 06/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index eca120b..d4ccc80 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -29,30 +29,30 @@ jobs: DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} - name: Run SLO - uses: ydb-platform/slo-tests@k8s-local -# if: env.DOCKER_REPO != null -# env: -# DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + uses: ydb-platform/slo-tests@main + if: env.DOCKER_REPO != null + env: + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} continue-on-error: true with: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# KUBECONFIG_B64: ${{ secrets.SLO_KUBE_CONFIG }} -# AWS_CREDENTIALS_B64: ${{ secrets.SLO_AWS_CREDENTIALS }} -# AWS_CONFIG_B64: ${{ secrets.SLO_AWS_CONFIG }} -# DOCKER_USERNAME: ${{ secrets.SLO_DOCKER_USERNAME }} -# DOCKER_PASSWORD: ${{ secrets.SLO_DOCKER_PASSWORD }} -# DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} -# DOCKER_FOLDER: ${{ secrets.SLO_DOCKER_FOLDER }} -# s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} -# s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} -# grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + KUBECONFIG_B64: ${{ secrets.SLO_KUBE_CONFIG }} + AWS_CREDENTIALS_B64: ${{ secrets.SLO_AWS_CREDENTIALS }} + AWS_CONFIG_B64: ${{ secrets.SLO_AWS_CONFIG }} + DOCKER_USERNAME: ${{ secrets.SLO_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SLO_DOCKER_PASSWORD }} + DOCKER_REPO: ${{ secrets.SLO_DOCKER_REPO }} + DOCKER_FOLDER: ${{ secrets.SLO_DOCKER_FOLDER }} + s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} + s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} + grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} # grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }} -# grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 -# grafana_dashboard_width: 2000 -# grafana_dashboard_height: 2300 -# ydb_version: 'newest' -# timeBetweenPhases: 30 -# shutdownTime: 30 + grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 + grafana_dashboard_width: 2000 + grafana_dashboard_height: 2300 + ydb_version: 'newest' + timeBetweenPhases: 30 + shutdownTime: 30 language_id0: 'hibernate' workload_path0: 'slo/hibernate' From 4020d0f543a01e4ad47795cb7a5df36c972ef19b Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 16:39:08 +0300 Subject: [PATCH 07/25] feat: SLO Hibernate --- slo/Dockerfile | 10 ++++++++++ slo/hibernate/Dockerfile | 11 ----------- slo/hibernate/pom.xml | 2 -- 3 files changed, 10 insertions(+), 13 deletions(-) create mode 100644 slo/Dockerfile delete mode 100644 slo/hibernate/Dockerfile diff --git a/slo/Dockerfile b/slo/Dockerfile new file mode 100644 index 0000000..d1057da --- /dev/null +++ b/slo/Dockerfile @@ -0,0 +1,10 @@ +FROM maven:3.8.4-openjdk-17 AS build +ARG SRC_PATH +COPY . . +WORKDIR /${SRC_PATH} +RUN mvn package -DskipTests + +FROM openjdk:21 AS run +WORKDIR /app +COPY --from=build . . +ENTRYPOINT ["java", "-jar", "./hibernate/target/hibernate-0.0.1-SNAPSHOT.jar"] \ No newline at end of file diff --git a/slo/hibernate/Dockerfile b/slo/hibernate/Dockerfile deleted file mode 100644 index 3a2e0e0..0000000 --- a/slo/hibernate/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM maven:3.8.4-openjdk-17 AS build -WORKDIR /app -COPY pom.xml . -RUN mvn dependency:resolve -COPY src ./src -RUN mvn package -DskipTests - -FROM openjdk:17 AS run -WORKDIR /app -COPY --from=build /app/target/hibernate-0.0.1-SNAPSHOT.jar /app/app.jar -ENTRYPOINT ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/slo/hibernate/pom.xml b/slo/hibernate/pom.xml index e995e78..53d4940 100644 --- a/slo/hibernate/pom.xml +++ b/slo/hibernate/pom.xml @@ -48,7 +48,6 @@ spring - jpa @@ -61,5 +60,4 @@ - From bbe96286ff504151a91c41cd1c9a73f9bac446cd Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 16:40:31 +0300 Subject: [PATCH 08/25] feat: SLO Hibernate --- slo/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/slo/Dockerfile b/slo/Dockerfile index d1057da..df5e98b 100644 --- a/slo/Dockerfile +++ b/slo/Dockerfile @@ -1,7 +1,6 @@ FROM maven:3.8.4-openjdk-17 AS build -ARG SRC_PATH COPY . . -WORKDIR /${SRC_PATH} +WORKDIR /hibernate RUN mvn package -DskipTests FROM openjdk:21 AS run From 1fc4d107843e943cabb4fba78203cfb18ce858b3 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:10:37 +0300 Subject: [PATCH 09/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 2 +- slo/Dockerfile | 11 ++++++----- slo/hibernate/pom.xml | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index d4ccc80..79c9b9d 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -58,7 +58,7 @@ jobs: workload_path0: 'slo/hibernate' language0: 'Hibernate YQL Dialect over YDB JDBC Driver' workload_build_context0: ../.. - workload_build_options0: -f Dockerfile + workload_build_options0: -f Dockerfile --build-arg SRC_PATH=hibernate - uses: actions/upload-artifact@v3 diff --git a/slo/Dockerfile b/slo/Dockerfile index df5e98b..13ca347 100644 --- a/slo/Dockerfile +++ b/slo/Dockerfile @@ -1,9 +1,10 @@ FROM maven:3.8.4-openjdk-17 AS build -COPY . . -WORKDIR /hibernate +ARG SRC_PATH +COPY . /app +WORKDIR /app/${SRC_PATH} RUN mvn package -DskipTests +RUN cp -r ./target/* ./.. FROM openjdk:21 AS run -WORKDIR /app -COPY --from=build . . -ENTRYPOINT ["java", "-jar", "./hibernate/target/hibernate-0.0.1-SNAPSHOT.jar"] \ No newline at end of file +COPY --from=build /app . +ENTRYPOINT ["java", "-jar", "./slo.jar"] \ No newline at end of file diff --git a/slo/hibernate/pom.xml b/slo/hibernate/pom.xml index 53d4940..9ab25cb 100644 --- a/slo/hibernate/pom.xml +++ b/slo/hibernate/pom.xml @@ -33,6 +33,7 @@ + slo ${project.basedir}/src/main/kotlin From d071d2480cb6ee560c795c7c3dd07ca2bbcb0272 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:14:48 +0300 Subject: [PATCH 10/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index 79c9b9d..b51e02e 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -55,7 +55,7 @@ jobs: shutdownTime: 30 language_id0: 'hibernate' - workload_path0: 'slo/hibernate' + workload_path0: 'slo' language0: 'Hibernate YQL Dialect over YDB JDBC Driver' workload_build_context0: ../.. workload_build_options0: -f Dockerfile --build-arg SRC_PATH=hibernate From 917fc428e72fa09f7a0c135395efd9267a8fa3b9 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:19:23 +0300 Subject: [PATCH 11/25] feat: SLO Hibernate --- slo/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/slo/Dockerfile b/slo/Dockerfile index 13ca347..679242c 100644 --- a/slo/Dockerfile +++ b/slo/Dockerfile @@ -2,8 +2,10 @@ FROM maven:3.8.4-openjdk-17 AS build ARG SRC_PATH COPY . /app WORKDIR /app/${SRC_PATH} +RUN ls RUN mvn package -DskipTests RUN cp -r ./target/* ./.. +RUN ls .. FROM openjdk:21 AS run COPY --from=build /app . From 598c07e68325195bcd843b3e9bbc5a47cf3aa77f Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:26:16 +0300 Subject: [PATCH 12/25] feat: SLO Hibernate --- slo/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/slo/Dockerfile b/slo/Dockerfile index 679242c..d08891d 100644 --- a/slo/Dockerfile +++ b/slo/Dockerfile @@ -1,6 +1,7 @@ FROM maven:3.8.4-openjdk-17 AS build ARG SRC_PATH COPY . /app +RUN ls WORKDIR /app/${SRC_PATH} RUN ls RUN mvn package -DskipTests From 960e74272a3c25e7db9a57404bb806e3351db688 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:32:57 +0300 Subject: [PATCH 13/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index b51e02e..762bff7 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -57,7 +57,7 @@ jobs: language_id0: 'hibernate' workload_path0: 'slo' language0: 'Hibernate YQL Dialect over YDB JDBC Driver' - workload_build_context0: ../.. + workload_build_context0: workload_build_options0: -f Dockerfile --build-arg SRC_PATH=hibernate From a2ebda98860b87404240256a39650a7f21aae694 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 17:48:27 +0300 Subject: [PATCH 14/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 2 -- .../src/main/resources/{ => db}/migration/V1__create_table.sql | 0 2 files changed, 2 deletions(-) rename slo/hibernate/src/main/resources/{ => db}/migration/V1__create_table.sql (100%) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index 762bff7..4bd83e5 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -57,10 +57,8 @@ jobs: language_id0: 'hibernate' workload_path0: 'slo' language0: 'Hibernate YQL Dialect over YDB JDBC Driver' - workload_build_context0: workload_build_options0: -f Dockerfile --build-arg SRC_PATH=hibernate - - uses: actions/upload-artifact@v3 if: always() && env.DOCKER_REPO != null env: diff --git a/slo/hibernate/src/main/resources/migration/V1__create_table.sql b/slo/hibernate/src/main/resources/db/migration/V1__create_table.sql similarity index 100% rename from slo/hibernate/src/main/resources/migration/V1__create_table.sql rename to slo/hibernate/src/main/resources/db/migration/V1__create_table.sql From fd3f3376c8b6a7806e3f670c75e8f7e07c83270b Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 18:00:48 +0300 Subject: [PATCH 15/25] feat: SLO Hibernate --- .../{ydb/tech => tech/ydb/slo}/hibernate/HibernateApplication.kt | 0 .../{ydb/tech => tech/ydb/slo}/hibernate/entity/SloEntity.kt | 0 .../ydb/slo}/hibernate/repository/SloEntityRepository.kt | 0 .../{ydb/tech => tech/ydb/slo}/hibernate/retry/YdbRetryable.kt | 0 .../{ydb/tech => tech/ydb/slo}/hibernate/service/SloService.kt | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename slo/hibernate/src/main/kotlin/{ydb/tech => tech/ydb/slo}/hibernate/HibernateApplication.kt (100%) rename slo/hibernate/src/main/kotlin/{ydb/tech => tech/ydb/slo}/hibernate/entity/SloEntity.kt (100%) rename slo/hibernate/src/main/kotlin/{ydb/tech => tech/ydb/slo}/hibernate/repository/SloEntityRepository.kt (100%) rename slo/hibernate/src/main/kotlin/{ydb/tech => tech/ydb/slo}/hibernate/retry/YdbRetryable.kt (100%) rename slo/hibernate/src/main/kotlin/{ydb/tech => tech/ydb/slo}/hibernate/service/SloService.kt (100%) diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt similarity index 100% rename from slo/hibernate/src/main/kotlin/ydb/tech/hibernate/HibernateApplication.kt rename to slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/entity/SloEntity.kt similarity index 100% rename from slo/hibernate/src/main/kotlin/ydb/tech/hibernate/entity/SloEntity.kt rename to slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/entity/SloEntity.kt diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/repository/SloEntityRepository.kt similarity index 100% rename from slo/hibernate/src/main/kotlin/ydb/tech/hibernate/repository/SloEntityRepository.kt rename to slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/repository/SloEntityRepository.kt diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/retry/YdbRetryable.kt similarity index 100% rename from slo/hibernate/src/main/kotlin/ydb/tech/hibernate/retry/YdbRetryable.kt rename to slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/retry/YdbRetryable.kt diff --git a/slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/service/SloService.kt similarity index 100% rename from slo/hibernate/src/main/kotlin/ydb/tech/hibernate/service/SloService.kt rename to slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/service/SloService.kt From 01207b86ef4dab20dffb7bc258c60e6e209a5b00 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 18:17:44 +0300 Subject: [PATCH 16/25] feat: SLO Hibernate --- .github/workflows/slo.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/slo.yaml b/.github/workflows/slo.yaml index 4bd83e5..c46207a 100644 --- a/.github/workflows/slo.yaml +++ b/.github/workflows/slo.yaml @@ -46,8 +46,7 @@ jobs: s3_endpoint: ${{ secrets.SLO_S3_ENDPOINT }} s3_images_folder: ${{ vars.SLO_S3_IMAGES_FOLDER }} grafana_domain: ${{ vars.SLO_GRAFANA_DOMAIN }} - # grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }} - grafana_dashboard: dca60386-0d3d-43f5-a2af-5f3fd3e3b295 + grafana_dashboard: ${{ vars.SLO_GRAFANA_DASHBOARD }} grafana_dashboard_width: 2000 grafana_dashboard_height: 2300 ydb_version: 'newest' From fa43cd67a962088640b89b9586f0c6f6793fa45c Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Thu, 29 Aug 2024 18:42:13 +0300 Subject: [PATCH 17/25] feat: SLO Hibernate --- .../kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt index 2498fcc..96dda17 100644 --- a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt +++ b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt @@ -19,10 +19,6 @@ class HibernateApplication : CommandLineRunner { private lateinit var sloService: SloService override fun run(vararg args: String) { - if (args.isNotEmpty() && args[0] == "create") { - return // Flyway created slo table - } - val pushGateway = PushGateway.builder() .job("hibernate-slo") .address("prometheus-pushgateway:9091") From 69e8bbe83bc89fc1a598797e6fb155c0456c5ab6 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 11:42:18 +0300 Subject: [PATCH 18/25] feat: SLO Hibernate --- slo/docker-compose.yml | 59 +++++++++++++++++++ .../ydb/slo/hibernate/HibernateApplication.kt | 4 +- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/slo/docker-compose.yml b/slo/docker-compose.yml index 29f465b..a2843a3 100644 --- a/slo/docker-compose.yml +++ b/slo/docker-compose.yml @@ -46,3 +46,62 @@ services: - "3000:3000" networks: - monitor-net + ydb: + image: cr.yandex/yc/yandex-docker-local-ydb:latest + container_name: ydb + environment: + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_DEFAULT_LOG_LEVEL=NOTICE + ports: + - "2135:2135" + - "2136:2136" + - "8765:8765" + volumes: + - ../data/ydb_certs:/ydb_certs + networks: + - monitor-net + + slo-create: + build: + dockerfile: ./Dockerfile + args: + - SRC_PATH=hibernate + command: + - 'create' + - 'http://ydb:2136' + - '/local' + - '--table-name' + - 'slo-dotnet' + - '--min-partitions-count' + - '6' + - '--max-partitions-count' + - '1000' + - '--partition-size' + - '1' + - '--initial-data-count' + - '1000' + networks: + - monitor-net + depends_on: + ydb: + condition: service_healthy + + slo-run: + build: + dockerfile: ./Dockerfile + command: + - 'run' + - 'http://ydb:2136' + - '/local' + - '--prom-pgw' + - 'http://prometheus-pushgateway:9091' + - '--table-name' + - 'slo-dotnet' + networks: + - monitor-net + depends_on: + slo-create: + condition: service_completed_successfully \ No newline at end of file diff --git a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt index 96dda17..6e3408d 100644 --- a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt +++ b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt @@ -20,7 +20,7 @@ class HibernateApplication : CommandLineRunner { override fun run(vararg args: String) { val pushGateway = PushGateway.builder() - .job("hibernate-slo") + .job("slo") .address("prometheus-pushgateway:9091") .build() @@ -45,7 +45,6 @@ class HibernateApplication : CommandLineRunner { okCount.labelValues("read", "TODO", "hibernate").inc() } catch (e: Exception) { notOkCount.labelValues("read", "TODO", "hibernate").inc() - println(e) } pushGateway.push() } @@ -56,7 +55,6 @@ class HibernateApplication : CommandLineRunner { okCount.labelValues("write", "TODO", "hibernate").inc() } catch (e: Exception) { notOkCount.labelValues("write", "TODO", "hibernate").inc() - println(e) } pushGateway.push() } From acd059ee9b384b48c2896e180fb3f3ab4cdddcd9 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 11:54:06 +0300 Subject: [PATCH 19/25] feat: SLO Hibernate --- .../main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt index 6e3408d..e41571a 100644 --- a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt +++ b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt @@ -20,7 +20,7 @@ class HibernateApplication : CommandLineRunner { override fun run(vararg args: String) { val pushGateway = PushGateway.builder() - .job("slo") + .job("slo-hibernate") .address("prometheus-pushgateway:9091") .build() From dbd64af4f46178346531715804d6b51b3e830597 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 12:04:37 +0300 Subject: [PATCH 20/25] feat: SLO Hibernate --- slo/hibernate/src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/slo/hibernate/src/main/resources/application.properties b/slo/hibernate/src/main/resources/application.properties index 24114da..f7b2d89 100644 --- a/slo/hibernate/src/main/resources/application.properties +++ b/slo/hibernate/src/main/resources/application.properties @@ -3,3 +3,4 @@ spring.datasource.url=jdbc:ydb:grpc://database-sample-grpc:2135/root/database-sa spring.jpa.properties.hibernate.dialect=tech.ydb.hibernate.dialect.YdbDialect spring.datasource.driver-class-name=tech.ydb.jdbc.YdbDriver +spring.flyway.connect-retries=10 From cd2950485920deaa95fa20060cd2056ad8e67591 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 12:26:38 +0300 Subject: [PATCH 21/25] feat: SLO Hibernate --- .../main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt index e41571a..a71f161 100644 --- a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt +++ b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt @@ -20,7 +20,7 @@ class HibernateApplication : CommandLineRunner { override fun run(vararg args: String) { val pushGateway = PushGateway.builder() - .job("slo-hibernate") + .job("workload-hibernate") .address("prometheus-pushgateway:9091") .build() From 18a575d40be5926561243357164ad0f6679bfb67 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 13:08:04 +0300 Subject: [PATCH 22/25] feat: SLO Hibernate --- .../kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt | 4 ++++ slo/hibernate/src/main/resources/application.properties | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt index a71f161..3544642 100644 --- a/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt +++ b/slo/hibernate/src/main/kotlin/tech/ydb/slo/hibernate/HibernateApplication.kt @@ -19,6 +19,10 @@ class HibernateApplication : CommandLineRunner { private lateinit var sloService: SloService override fun run(vararg args: String) { + if (args[0] == "create") { + return + } + val pushGateway = PushGateway.builder() .job("workload-hibernate") .address("prometheus-pushgateway:9091") diff --git a/slo/hibernate/src/main/resources/application.properties b/slo/hibernate/src/main/resources/application.properties index f7b2d89..6cf18aa 100644 --- a/slo/hibernate/src/main/resources/application.properties +++ b/slo/hibernate/src/main/resources/application.properties @@ -3,4 +3,4 @@ spring.datasource.url=jdbc:ydb:grpc://database-sample-grpc:2135/root/database-sa spring.jpa.properties.hibernate.dialect=tech.ydb.hibernate.dialect.YdbDialect spring.datasource.driver-class-name=tech.ydb.jdbc.YdbDriver -spring.flyway.connect-retries=10 +spring.flyway.connect-retries=1000 From 96b3d2fe23272679ef0505d4e526beb716c32e86 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Fri, 30 Aug 2024 13:19:09 +0300 Subject: [PATCH 23/25] feat: SLO Hibernate --- slo/hibernate/src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/slo/hibernate/src/main/resources/application.properties b/slo/hibernate/src/main/resources/application.properties index 6cf18aa..2ca692f 100644 --- a/slo/hibernate/src/main/resources/application.properties +++ b/slo/hibernate/src/main/resources/application.properties @@ -4,3 +4,4 @@ spring.jpa.properties.hibernate.dialect=tech.ydb.hibernate.dialect.YdbDialect spring.datasource.driver-class-name=tech.ydb.jdbc.YdbDriver spring.flyway.connect-retries=1000 +spring.flyway.connect-retries-interval=5s \ No newline at end of file From f74fc4dbaf144125fa85d75f9d827a09240c1bf5 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Wed, 2 Oct 2024 13:41:16 +0300 Subject: [PATCH 24/25] fix --- liquibase-dialect/pom.xml | 2 +- slo/hibernate/src/main/resources/application.properties | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/liquibase-dialect/pom.xml b/liquibase-dialect/pom.xml index a5b0826..2af853a 100644 --- a/liquibase-dialect/pom.xml +++ b/liquibase-dialect/pom.xml @@ -43,7 +43,7 @@ 1.8 4.24.0 - 2.2.0 + 2.2.6 diff --git a/slo/hibernate/src/main/resources/application.properties b/slo/hibernate/src/main/resources/application.properties index 2ca692f..16fd831 100644 --- a/slo/hibernate/src/main/resources/application.properties +++ b/slo/hibernate/src/main/resources/application.properties @@ -3,5 +3,5 @@ spring.datasource.url=jdbc:ydb:grpc://database-sample-grpc:2135/root/database-sa spring.jpa.properties.hibernate.dialect=tech.ydb.hibernate.dialect.YdbDialect spring.datasource.driver-class-name=tech.ydb.jdbc.YdbDriver -spring.flyway.connect-retries=1000 -spring.flyway.connect-retries-interval=5s \ No newline at end of file +spring.flyway.connect-retries=10 +spring.flyway.connect-retries-interval=1s \ No newline at end of file From 93f0bdd60360d5a67929d63653153486ac6886f7 Mon Sep 17 00:00:00 2001 From: Kirill Kurdyukov Date: Wed, 2 Oct 2024 13:49:41 +0300 Subject: [PATCH 25/25] update liquibase jdbc driver version --- liquibase-dialect/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liquibase-dialect/pom.xml b/liquibase-dialect/pom.xml index 2af853a..e195760 100644 --- a/liquibase-dialect/pom.xml +++ b/liquibase-dialect/pom.xml @@ -43,7 +43,7 @@ 1.8 4.24.0 - 2.2.6 + 2.2.8