Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

迁移数据库的时候,创建语句中有转义符,迁移尚未识别到 #324

Open
huoxiaojun2009 opened this issue Feb 18, 2025 · 1 comment
Labels
bug Something isn't working

Comments

@huoxiaojun2009
Copy link

./ape-dts task_config.ini.struct

2025-02-18T16:35:21.787408100+08:00 INFO default_logger - ddl begin: CREATE TABLE IF NOT EXISTS dbops.t_db_data_sync_task (id bigint(20) auto_increment COMMENT '主键ID' NOT NULL ,creator_id bigint(20) unsigned NOT NULL ,creator varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL ,create_time datetime NOT NULL ,update_time datetime NOT NULL ,modifier varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL ,modifier_id bigint(20) unsigned NOT NULL ,task_name varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务名' NOT NULL ,task_cron varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务Cron表达式' NOT NULL ,src_db_id bigint(20) COMMENT '源数据库ID' NOT NULL ,src_db_name varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库名' NULL ,src_tag_path varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库tag路径' NULL ,target_db_id bigint(20) COMMENT '目标数据库ID' NOT NULL ,target_db_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库名' NULL ,target_tag_path varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库tag路径' NULL ,target_table_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库表名' NULL ,data_sql text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '数据查询sql' NOT NULL ,page_size bigint(20) COMMENT '数据同步分页大小' NOT NULL ,upd_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '更新字段,默认'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '当前更新值' NULL ,id_rule tinyint(2) DEFAULT '1' COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' NOT NULL ,pk_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '主键id字段名,默认"id"' NULL ,field_map text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '字段映射json' NULL ,is_deleted tinyint(8) DEFAULT '0' NULL ,delete_time datetime NULL ,status tinyint(4) DEFAULT '1' COMMENT '状态 1启用 2禁用' NOT NULL ,recent_state tinyint(4) DEFAULT '0' COMMENT '最近执行状态 1成功 -1失败' NOT NULL ,task_key varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务唯一标识' NULL ,running_state tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止' NULL ,duplicate_strategy tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' NULL ,upd_field_src varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '更新值来源, 如select name as user_name from user' NULL , PRIMARY KEY (id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='数据同步'
2025-02-18T16:35:21.788819840+08:00 ERROR default_logger - ddl failed, error: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1
2025-02-18T16:35:21.832808290+08:00 ERROR default_logger - panic: panicked at /project/dt-task/src/task_runner.rs:326:36:
called Result::unwrap() on an Err value: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1

Caused by:
0: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1
1: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1

Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: dt_connector::sinker::base_struct_sinker::BaseStructSinker::sink_structs::{{closure}}
2: <dt_connector::sinker::mysql::mysql_struct_sinker::MysqlStructSinker as dt_connector::Sinker>::sink_struct::{{closure}}
3: <dt_parallelizer::serial_parallelizer::SerialParallelizer as dt_parallelizer::Parallelizer>::sink_struct::{{closure}}
4: <dt_pipeline::base_pipeline::BasePipeline as dt_pipeline::Pipeline>::start::{{closure}}
5: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
6: tokio::runtime::task::raw::poll
7: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
8: tokio::runtime::scheduler::multi_thread::worker::Context::run
9: tokio::runtime::context::runtime::enter_runtime
10: tokio::runtime::scheduler::multi_thread::worker::run
11: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
12: tokio::runtime::task::core::Core<T,S>::poll
13: tokio::runtime::task::harness::Harness<T,S>::poll
14: tokio::runtime::blocking::pool::Inner::run
15: std::sys::backtrace::__rust_begin_short_backtrace
16: core::ops::function::FnOnce::call_once{{vtable.shim}}
17: std::sys::pal::unix::thread::Thread::new::thread_start
18: start_thread
at /data/software/huoxj/glibc-2.18/nptl/pthread_create.c:309
19: __clone
at /data/software/huoxj/glibc-2.18/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111
2025-02-18T16:35:21.833140898+08:00 ERROR default_logger - panic: panicked at /project/dt-task/src/task_runner.rs:341:31:
called Result::unwrap() on an Err value: JoinError::Panic(Id(18), "called Result::unwrap() on an Err value: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n\nCaused by:\n 0: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n 1: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n\nStack backtrace:\n 0: anyhow::error::<impl core::convert::From for anyhow::Error>::from\n 1: dt_connector::sinker::base_struct_sinker::BaseStructSinker::sink_structs::{{closure}}\n 2: <dt_connector::sinker::mysql::mysql_struct_sinker::MysqlStructSinker as dt_connector::Sinker>::sink_struct::{{closure}}\n 3: <dt_parallelizer::serial_parallelizer::SerialParallelizer as dt_parallelizer::Parallelizer>::sink_struct::{{closure}}\n 4: <dt_pipeline::base_pipeline::BasePipeline as dt_pipeline::Pipeline>::start::{{closure}}\n 5: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}\n 6: tokio::runtime::task::raw::poll\n 7: tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n 8: tokio::runtime::scheduler::multi_thread::worker::Context::run\n 9: tokio::runtime::context::runtime::enter_runtime\n 10: tokio::runtime::scheduler::multi_thread::worker::run\n 11: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll\n 12: tokio::runtime::task::core::Core<T,S>::poll\n 13: tokio::runtime::task::harness::Harness<T,S>::poll\n 14: tokio::runtime::blocking::pool::Inner::run\n 15: std::sys::backtrace::__rust_begin_short_backtrace\n 16: core::ops::function::FnOnce::call_once{{vtable.shim}}\n 17: std::sys::pal::unix::thread::Thread::new::thread_start\n 18: start_thread\n at /data/software/huoxj/glibc-2.18/nptl/pthread_create.c:309\n 19: __clone\n at /data/software/huoxj/glibc-2.18/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111", ...)
[root@oraem dts]#

源table创建语句
CREATE TABLE t_db_data_sync_task (
id BIGINT AUTO_INCREMENT NOT NULL COMMENT '主键ID' ,
creator_id BIGINT UNSIGNED NOT NULL,
creator VARCHAR(32) NOT NULL,
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
modifier VARCHAR(32) NOT NULL,
modifier_id BIGINT UNSIGNED NOT NULL,
task_name VARCHAR(255) NOT NULL COMMENT '任务名' ,
task_cron VARCHAR(50) NOT NULL COMMENT '任务Cron表达式' ,
src_db_id BIGINT NOT NULL COMMENT '源数据库ID' ,
src_db_name VARCHAR(100) NULL COMMENT '源数据库名' ,
src_tag_path VARCHAR(200) NULL COMMENT '源数据库tag路径' ,
target_db_id BIGINT NOT NULL COMMENT '目标数据库ID' ,
target_db_name VARCHAR(150) NULL COMMENT '目标数据库名' ,
target_tag_path VARCHAR(255) NULL COMMENT '目标数据库tag路径' ,
target_table_name VARCHAR(150) NULL COMMENT '目标数据库表名' ,
data_sql TEXT NOT NULL COMMENT '数据查询sql' ,
page_size BIGINT NOT NULL COMMENT '数据同步分页大小' ,
upd_field VARCHAR(100) NOT NULL DEFAULT 'id' COMMENT '更新字段,默认'id'' ,
upd_field_val VARCHAR(100) NULL COMMENT '当前更新值' ,
id_rule TINYINT NOT NULL DEFAULT 1 COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' ,
pk_field VARCHAR(100) NULL DEFAULT 'id' COMMENT '主键id字段名,默认"id"' ,
field_map TEXT NULL COMMENT '字段映射json' ,
is_deleted TINYINT NULL DEFAULT 0 ,
delete_time DATETIME NULL,
status TINYINT NOT NULL DEFAULT 1 COMMENT '状态 1启用 2禁用' ,
recent_state TINYINT NOT NULL DEFAULT 0 COMMENT '最近执行状态 1成功 -1失败' ,
task_key VARCHAR(100) NULL COMMENT '任务唯一标识' ,
running_state TINYINT NULL DEFAULT 2 COMMENT '运行时状态 1运行中、2待运行、3已停止' ,
duplicate_strategy TINYINT NULL DEFAULT -1 COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' ,
upd_field_src VARCHAR(191) NULL COMMENT '更新值来源, 如select name as user_name from user' ,
PRIMARY KEY (id)
)
ENGINE = InnoDB
COMMENT = '数据同步';

主要是问题是,在create 语句中使用了 upd_field VARCHAR(100) NOT NULL DEFAULT 'id' COMMENT '更新字段,默认'id'' , 做转义,在做数据迁移的时候没有识别到。
ape_dts日志中创建数据库的sql:
CREATE TABLE IF NOT EXISTS dbops.t_db_data_sync_task (
id bigint(20) auto_increment COMMENT '主键ID' NOT NULL,
creator_id bigint(20) unsigned NOT NULL,
creator varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
create_time datetime NOT NULL,
update_time datetime NOT NULL,
modifier varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
modifier_id bigint(20) unsigned NOT NULL,
task_name varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务名' NOT NULL,
task_cron varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务Cron表达式' NOT NULL,
src_db_id bigint(20) COMMENT '源数据库ID' NOT NULL,
src_db_name varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库名' NULL,
src_tag_path varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库tag路径' NULL,
target_db_id bigint(20) COMMENT '目标数据库ID' NOT NULL,
target_db_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库名' NULL,
target_tag_path varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库tag路径' NULL,
target_table_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库表名' NULL,
data_sql text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '数据查询sql' NOT NULL,
page_size bigint(20) COMMENT '数据同步分页大小' NOT NULL,
upd_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '更新字段,默认' id '' NOT NULL,
upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '当前更新值' NULL,
id_rule tinyint(2) DEFAULT '1' COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' NOT NULL,
pk_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '主键id字段名,默认"id"' NULL,
field_map text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '字段映射json' NULL,
is_deleted tinyint(8) DEFAULT '0' NULL,
delete_time datetime NULL,
status tinyint(4) DEFAULT '1' COMMENT '状态 1启用 2禁用' NOT NULL,
recent_state tinyint(4) DEFAULT '0' COMMENT '最近执行状态 1成功 -1失败' NOT NULL,
task_key varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务唯一标识' NULL,
running_state tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止' NULL,
duplicate_strategy tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' NULL,
upd_field_src varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '更新值来源, 如select name as user_name from user' NULL,
PRIMARY KEY (id)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci COMMENT = '数据同步'

@huoxiaojun2009 huoxiaojun2009 added the bug Something isn't working label Feb 18, 2025
@qianyiwen2019
Copy link
Collaborator

OK,我们看下这个问题

# for free to join this conversation on GitHub. Already have an account? # to comment
Labels
bug Something isn't working
Projects
None yet
Development

No branches or pull requests

2 participants