清理django_q_task表以后工单无法成功执行

[tanglu86]

重现步骤

我们公司部署的Archery已经运行了2年多，django_q_task表数据量过大，目前有50多G，在测试环境对该表truncate后发现所有工单都无法执行成功，均为排队中的状态，尝试过以下排错无果：
1、archery.log、qcluster.log、supervisord.log 三个日志没有报错信息
2、goInception中通过inception show processlist看不到工单
3、Failed task里没有失败的任务
4、python manage.py qmonitor是正常RUNNING状态

预期外的结果

日志文本

No response

版本

1.8.4

部署方式

手工部署

是否还有其他可以辅助定位问题的信息？比如数据库版本等

MySQL: 5.7.30
Browsers: Chrome
goInception: v1.3.0-42

[nick2wang]

清空django_q_task并不会影响后续任务的执行，看下debug信息里的qcluser状态
http://your_archery_host/api/debug

{
    "archery": {
        "version": "1.9.1"
    },
    "django_q": {
        "version": "1.3.9",
        "conf": {
            "name": "archery",
            "workers": 4,
            "recycle": 500,
            "timeout": 60,
            "compress": true,
            "cpu_affinity": 1,
            "save_limit": 0,
            "queue_limit": 50,
            "label": "Django Q",
            "django_redis": "default",
            "sync": false
        },
        "q_cluster_stats": [
            {
                "host": "test",
                "cluster_id": "xxx",
                "state": "Idle",
                "pool": 2,
                "tq": 0,
                "rq": 0,
                "rc": 0,
                "up": "5:17:19"
            }
        ],
        "q_broker_stats": {
            "info": "Redis 5.0.13",
            "Queued": 0,
            "Success": 274,
            "Failures": 1
        }
    }
}

[tanglu86]

q_cluster_stats

{
"archery": {
"version": "1.8.4"
},
"django_q": {
"version": "1.3.4",
"conf": {
"name": "archery",
"workers": 4,
"recycle": 30,
"timeout": 600,
"compress": true,
"cpu_affinity": 1,
"save_limit": 0,
"queue_limit": 20,
"label": "Django Q",
"django_redis": "default",
"sync": false
},
"q_cluster_stats": [{
"host": "archery-desensitization-test-ns-172-20-0-236",
"cluster_id": "88d88cca-6140-4299-9b1d-fd2b5877d881",
"state": "Working",
"pool": 4,
"tq": 20,
"rq": 68,
"rc": 19,
"up": "1:03:50"
}],
"q_broker_stats": {
"info": "Redis 3.2.9",
"Queued": 1019,
"Success": 1107,
"Failures": 1
}
},
"inception": {
"enable_inception": null,
"inception_info": "\u83b7\u53d6Inception\u4fe1\u606f\u62a5\u9519:connect() argument 1 must be str, not None",
"goinception_info": {
"version": "v1.3.0-42-g169e116",
"max_allowed_packet": "4194304",
"lang": "zh_cn",
"osc_on": "false",
"osc_bin_dir": "/usr/local/bin",
"ghost_on": "false"
},
"backup_info": "\u65e0\u6cd5\u8fde\u63a5Inception\u5907\u4efd\u5e93\n(1045, "Access denied for user 'it'@'archery-desensitization-test-ns-172-20-0-236' (using password: YES)")"
},
"runtime_info": {
"python_version": "3.9.10",
"mysql_info": {
"mysql_server_info": "5.7.30-log",
"timezone_name": "Asia/Shanghai"
},
"redis_info": {
"redis_version": "3.2.9",
"redis_mode": "standalone",
"role": "master",
"maxmemory_human": "0B",
"used_memory_human": "315.34M"
},
"sys_argv": ["/root/venv4archery184/bin/gunicorn", "-w", "2", "-b", "127.0.0.1:8000", "--timeout", "600", "archery.wsgi:application"],
"platform": ["Linux", "archery-desensitization-test-ns-172-20-0-236", "4.4.180-1.el7.elrepo.x86_64", "#1 SMP Thu May 16 17:40:04 EDT 2019", "x86_64", "x86_64"]
},
"sys_config": {
"admin_query_limit": "1000",
"api_user_whitelist": "",
"archery_base_url": "",
"auto_review": "",
"auto_review_db_type": "",
"auto_review_max_update_rows": "",
"auto_review_regex": "",
"auto_review_tag": "",
"auto_review_wrong": "",
"binlog2sql": "",
"critical_ddl_regex": "",
"data_masking": true,
"ddl_notify_auth_group": "",
"ddl_threshold": "",
"default_auth_group": "",
"default_resource_group": "\u6d4b\u8bd5",
"ding": "",
"ding_agent_id": "",
"ding_app_key": "",
"ding_app_secret": "",
"ding_archery_username": "",
"ding_dept_ids": "",
"ding_to_person": "",
"disable_star": "",
"dml_threshold": "100",
"enable_backup_switch": "",
"feishu": "",
"feishu_appid": "",
"feishu_app_secret": "",
"feishu_webhook": "",
"go_inception_host": "172.20.0.236",
"go_inception_port": "4000",
"inception_remote_backup_host": "172.20.0.236",
"inception_remote_backup_password": "",
"inception_remote_backup_port": "3306",
"inception_remote_backup_user": "it",
"index_path_url": "",
"lock_cnt_threshold": "",
"lock_time_threshold": "",
"mail": "",
"mail_smtp_password": "",
"mail_smtp_port": "",
"mail_smtp_server": "",
"mail_smtp_user": "",
"mail_ssl": "",
"manual": "",
"max_execution_time": "",
"my2sql": "",
"notify_phase_control": "Apply,Pass,Execute,Cancel",
"query_check": true,
"qywx_webhook": "",
"sign_up_enabled": true,
"soar": "",
"soar_test_dsn": "",
"sqladvisor": "",
"watermark_enabled": "",
"wx": "",
"wx_agent_id": "",
"wx_app_secret": "",
"wx_corpid": ""
},
"packages": ["aliyun-python-sdk-core-v3==2.13.33", "aliyun-python-sdk-rds==2.1.1", "arrow==1.2.3", "asgiref==3.6.0", "async-timeout==4.0.2", "attrs==22.2.0", "bcrypt==4.0.1", "blessed==1.19.1", "certifi==2022.12.7", "cffi==1.15.1", "chardet==3.0.4", "click==7.1.2", "clickhouse-driver==0.2.3", "cryptography==39.0.0", "cx-oracle==7.3.0", "django-auth-ldap==2.2.0", "django-filter==21.1", "django-mirage-field==1.1.6", "django-picklefield==3.0.1", "django-q==1.3.4", "django-redis==4.12.1", "django==3.1.14", "djangorestframework-simplejwt==4.3.0", "djangorestframework==3.12.4", "drf-spectacular==0.22.0", "future==0.18.2", "gunicorn==20.0.4", "h11==0.14.0", "httptools==0.1.1", "idna==2.10", "inflection==0.5.1", "jinja2==3.1.2", "jmespath==0.10.0", "jsonschema==4.17.3", "markupsafe==2.1.1", "mybatis-mapper2sql==0.1.9", "mysql-replication==0.22", "mysqlclient==2.0.1", "numpy==1.24.1", "pandas==1.1.5", "paramiko==2.12.0", "parsedatetime==2.4", "phoenixdb==0.7", "pillow==8.4.0", "pip==22.3.1", "prettytable==3.5.0", "protobuf==4.21.12", "psycopg2-binary==2.8.6", "pyasn1-modules==0.2.8", "pyasn1==0.4.8", "pycparser==2.21", "pycryptodome==3.10.1", "pyecharts==1.7.1", "pyjwt==1.7.1", "pymongo==3.11.0", "pymysql==0.9.3", "pynacl==1.5.0", "pyodbc==4.0.30", "pyodps==0.10.7.1", "pyotp==2.6.0", "pyrsistent==0.19.3", "python-dateutil==2.8.1", "python-ldap==3.4.3", "pytz-deprecation-shim==0.1.0.post0", "pytz==2022.7", "pyyaml==6.0", "qrcode==7.3.1", "redis==4.4.0", "requests==2.24.0", "schema-object==0.5.11", "schema-sync==0.9.7", "setuptools==65.6.3", "simplejson==3.17.2", "six==1.16.0", "sqlparse==0.4.3", "sshtunnel==0.1.5", "supervisor==4.1.0", "tqdm==4.64.1", "tzdata==2022.7", "tzlocal==4.2", "uritemplate==4.1.1", "urllib3==1.25.11", "uvicorn==0.12.2", "uvloop==0.14.0", "wcwidth==0.2.5", "wheel==0.38.4"]
}

[hhyo]

Queued这么多，都是排队状态，worker消费不过来，可以增加worker数，真不行可以清一下队列重新执行

[tanglu86]

调查清楚了，我们自己增加的自动脱敏任务创建了太多任务导致的，谢谢作者