From 668a87764f513bda6a50578abaa20f3f221bb6a9 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Sat, 22 Feb 2025 14:13:45 +0100 Subject: [PATCH 1/9] fix parameter binding order (#825) * use numbered parameters in sqlite * implement parameter deduplication * implement parameter re-duplication for mysql * Improve error messages on invalid sqlpage function calls. The messages now contain actionable advice. * simplify extract_set_variable --- CHANGELOG.md | 6 + src/webserver/database/mod.rs | 17 +- src/webserver/database/sql.rs | 352 +++++++++++++----- .../it_works_case_variables.sql | 10 + 4 files changed, 290 insertions(+), 95 deletions(-) create mode 100644 tests/sql_test_files/it_works_case_variables.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 024445b4..a7d28f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ - Fix a bug with date sorting in the table component. - Center table descriptions. - Fix a rare crash on startup in some restricted linux environments. +- Fix a rare but serious issue when on SQLite and MySQL, some variable values were assigned incorrectly + - `CASE WHEN $a THEN $x WHEN $b THEN $y` would be executed as `CASE WHEN $a THEN $b WHEN $x THEN $y` on these databases. + - the issue only occured when using in case expressions where variables were used both in conditions and results. +- Implement parameter deduplication. + Now, when you write `select $x where $x is not null`, the value of `$x` is sent to the database only once. It used to be sent as many times as `$x` appeared in the statement. +- Improve error messages on invalid sqlpage function calls. The messages now contain actionable advice. ## 0.33.0 (2025-02-15) diff --git a/src/webserver/database/mod.rs b/src/webserver/database/mod.rs index 344bbcc0..e9f0949e 100644 --- a/src/webserver/database/mod.rs +++ b/src/webserver/database/mod.rs @@ -9,7 +9,9 @@ mod syntax_tree; mod error_highlighting; mod sql_to_json; -pub use sql::{make_placeholder, ParsedSqlFile}; +pub use sql::ParsedSqlFile; +use sql::{DbPlaceHolder, DB_PLACEHOLDERS}; +use sqlx::any::AnyKind; pub struct Database { pub connection: sqlx::AnyPool, @@ -34,3 +36,16 @@ impl std::fmt::Display for Database { write!(f, "{:?}", self.connection.any_kind()) } } + +#[inline] +#[must_use] +pub fn make_placeholder(db_kind: AnyKind, arg_number: usize) -> String { + if let Some((_, placeholder)) = DB_PLACEHOLDERS.iter().find(|(kind, _)| *kind == db_kind) { + match *placeholder { + DbPlaceHolder::PrefixedNumber { prefix } => format!("{prefix}{arg_number}"), + DbPlaceHolder::Positional { placeholder } => placeholder.to_string(), + } + } else { + unreachable!("missing db_kind: {db_kind:?} in DB_PLACEHOLDERS ({DB_PLACEHOLDERS:?})") + } +} diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index df422fc1..8f92f11c 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -10,7 +10,8 @@ use sqlparser::ast::helpers::attached_token::AttachedToken; use sqlparser::ast::{ BinaryOperator, CastKind, CharacterLength, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, - OneOrManyWithParens, SelectItem, SetExpr, Spanned, Statement, Value, VisitMut, VisitorMut, + OneOrManyWithParens, SelectItem, SetExpr, Spanned, Statement, Value, Visit, VisitMut, Visitor, + VisitorMut, }; use sqlparser::dialect::{Dialect, MsSqlDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect}; use sqlparser::parser::{Parser, ParserError}; @@ -143,6 +144,26 @@ fn parse_sql<'a>( })) } +fn transform_to_positional_placeholders(stmt: &mut StmtWithParams, db_kind: AnyKind) { + if let Some((_, DbPlaceHolder::Positional { placeholder })) = + DB_PLACEHOLDERS.iter().find(|(kind, _)| *kind == db_kind) + { + let mut new_params = Vec::new(); + let mut query = stmt.query.clone(); + while let Some(pos) = query.find(TEMP_PLACEHOLDER_PREFIX) { + let start_of_number = pos + TEMP_PLACEHOLDER_PREFIX.len(); + let end = query[start_of_number..] + .find(|c: char| !c.is_ascii_digit()) + .map_or(query.len(), |i| start_of_number + i); + let param_idx = query[start_of_number..end].parse::().unwrap_or(1) - 1; + query.replace_range(pos..end, placeholder); + new_params.push(stmt.params[param_idx].clone()); + } + stmt.query = query; + stmt.params = new_params; + } +} + fn parse_single_statement( parser: &mut Parser<'_>, db_kind: AnyKind, @@ -161,8 +182,8 @@ fn parse_single_statement( semicolon = true; } let mut params = ParameterExtractor::extract_parameters(&mut stmt, db_kind); - if let Some((variable, value)) = extract_set_variable(&mut stmt, &mut params, db_kind) { - return Some(ParsedStatement::SetVariable { variable, value }); + if let Some(parsed) = extract_set_variable(&mut stmt, &mut params, db_kind) { + return Some(parsed); } if let Some(csv_import) = extract_csv_copy_statement(&mut stmt) { return Some(ParsedStatement::CsvImport(csv_import)); @@ -172,20 +193,26 @@ fn parse_single_statement( return Some(ParsedStatement::StaticSimpleSelect(static_statement)); } let delayed_functions = extract_toplevel_functions(&mut stmt); - remove_invalid_function_calls(&mut stmt, &mut params); + if let Err(err) = validate_function_calls(&stmt) { + return Some(ParsedStatement::Error(err.context(format!( + "Invalid SQLPage function call found in:\n{stmt}" + )))); + } let json_columns = extract_json_columns(&stmt, db_kind); let query = format!( "{stmt}{semicolon}", semicolon = if semicolon { ";" } else { "" } ); - log::debug!("Final transformed statement: {stmt}"); - Some(ParsedStatement::StmtWithParams(StmtWithParams { + let mut stmt_with_params = StmtWithParams { query, query_position: extract_query_start(&stmt), params, delayed_functions, json_columns, - })) + }; + transform_to_positional_placeholders(&mut stmt_with_params, db_kind); + log::debug!("Final transformed statement: {}", stmt_with_params.query); + Some(ParsedStatement::StmtWithParams(stmt_with_params)) } fn extract_query_start(stmt: &impl Spanned) -> SourceSpan { @@ -434,7 +461,7 @@ fn extract_set_variable( stmt: &mut Statement, params: &mut Vec, db_kind: AnyKind, -) -> Option<(StmtParam, StmtWithParams)> { +) -> Option { if let Statement::SetVariable { variables: OneOrManyWithParens::One(ObjectName(name)), value, @@ -451,18 +478,19 @@ fn extract_set_variable( let owned_expr = std::mem::replace(value, Expr::Value(Value::Null)); let mut select_stmt: Statement = expr_to_statement(owned_expr); let delayed_functions = extract_toplevel_functions(&mut select_stmt); - remove_invalid_function_calls(&mut select_stmt, params); + if let Err(err) = validate_function_calls(&select_stmt) { + return Some(ParsedStatement::Error(err)); + } let json_columns = extract_json_columns(&select_stmt, db_kind); - return Some(( - variable, - StmtWithParams { - query: select_stmt.to_string(), - query_position: extract_query_start(&select_stmt), - params: std::mem::take(params), - delayed_functions, - json_columns, - }, - )); + let mut value = StmtWithParams { + query: select_stmt.to_string(), + query_position: extract_query_start(&select_stmt), + params: std::mem::take(params), + delayed_functions, + json_columns, + }; + transform_to_positional_placeholders(&mut value, db_kind); + return Some(ParsedStatement::SetVariable { variable, value }); } } None @@ -473,9 +501,45 @@ struct ParameterExtractor { parameters: Vec, } -const PLACEHOLDER_PREFIXES: [(AnyKind, &str); 2] = - [(AnyKind::Postgres, "$"), (AnyKind::Mssql, "@p")]; -const DEFAULT_PLACEHOLDER: &str = "?"; +#[derive(Debug)] +pub enum DbPlaceHolder { + PrefixedNumber { prefix: &'static str }, + Positional { placeholder: &'static str }, +} + +pub const DB_PLACEHOLDERS: [(AnyKind, DbPlaceHolder); 4] = [ + ( + AnyKind::Sqlite, + DbPlaceHolder::PrefixedNumber { prefix: "?" }, + ), + ( + AnyKind::Postgres, + DbPlaceHolder::PrefixedNumber { prefix: "$" }, + ), + ( + AnyKind::MySql, + DbPlaceHolder::Positional { placeholder: "?" }, + ), + ( + AnyKind::Mssql, + DbPlaceHolder::PrefixedNumber { prefix: "@p" }, + ), +]; + +/// For positional parameters, we use a temporary placeholder during parameter extraction, +/// And then replace it with the actual placeholder during statement rewriting. +const TEMP_PLACEHOLDER_PREFIX: &str = "@SQLPAGE_TEMP"; + +fn get_placeholder_prefix(db_kind: AnyKind) -> &'static str { + if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = DB_PLACEHOLDERS + .iter() + .find(|(kind, _prefix)| *kind == db_kind) + { + prefix + } else { + TEMP_PLACEHOLDER_PREFIX + } +} impl ParameterExtractor { fn extract_parameters( @@ -490,15 +554,24 @@ impl ParameterExtractor { this.parameters } - fn make_placeholder(&self) -> Expr { - let name = make_placeholder(self.db_kind, self.parameters.len() + 1); - // We cast our placeholders to TEXT even though we always bind TEXT data to them anyway - // because that helps the database engine to prepare the query. - // For instance in PostgreSQL, the query planner will not be able to use an index on a - // column if the column is compared to a placeholder of type VARCHAR, but it will be able - // to use the index if the column is compared to a placeholder of type TEXT. + fn replace_with_placeholder(&mut self, value: &mut Expr, param: StmtParam) { + let placeholder = + if let Some(existing_idx) = self.parameters.iter().position(|p| *p == param) { + // Parameter already exists, use its index + self.make_placeholder_for_index(existing_idx + 1) + } else { + // New parameter, add it to the list + let placeholder = self.make_placeholder(); + log::trace!("Replacing {param} with {placeholder}"); + self.parameters.push(param); + placeholder + }; + *value = placeholder; + } + + fn make_placeholder_for_index(&self, index: usize) -> Expr { + let name = make_tmp_placeholder(self.db_kind, index); let data_type = match self.db_kind { - // MySQL requires CAST(? AS CHAR) and does not understand CAST(? AS TEXT) AnyKind::MySql => DataType::Char(None), AnyKind::Mssql => DataType::Varchar(Some(CharacterLength::Max)), _ => DataType::Text, @@ -512,38 +585,25 @@ impl ParameterExtractor { } } - fn handle_builtin_function( - &mut self, - func_name: &str, - mut arguments: Vec, - ) -> Expr { - #[allow(clippy::single_match_else)] - let placeholder = self.make_placeholder(); - let param = func_call_to_param(func_name, &mut arguments); - self.parameters.push(param); - placeholder + fn make_placeholder(&self) -> Expr { + self.make_placeholder_for_index(self.parameters.len() + 1) } fn is_own_placeholder(&self, param: &str) -> bool { - if let Some((_, prefix)) = PLACEHOLDER_PREFIXES - .iter() - .find(|(kind, _prefix)| *kind == self.db_kind) - { - if let Some(param) = param.strip_prefix(prefix) { - if let Ok(index) = param.parse::() { - return index <= self.parameters.len() + 1; - } + let prefix = get_placeholder_prefix(self.db_kind); + if let Some(param) = param.strip_prefix(prefix) { + if let Ok(index) = param.parse::() { + return index <= self.parameters.len() + 1; } - return false; } - param == DEFAULT_PLACEHOLDER + false } } -struct BadFunctionRemover; -impl VisitorMut for BadFunctionRemover { - type Break = StmtParam; - fn pre_visit_expr(&mut self, value: &mut Expr) -> ControlFlow { +struct InvalidFunctionFinder; +impl Visitor for InvalidFunctionFinder { + type Break = (String, Vec); + fn pre_visit_expr(&mut self, value: &Expr) -> ControlFlow { match value { Expr::Function(Function { name: ObjectName(func_name_parts), @@ -556,10 +616,8 @@ impl VisitorMut for BadFunctionRemover { .. }) if is_sqlpage_func(func_name_parts) => { let func_name = sqlpage_func_name(func_name_parts); - log::error!("Invalid function call to sqlpage.{func_name}. SQLPage function arguments must be static if the function is not at the top level of a select statement."); - let mut arguments = std::mem::take(args); - let param = func_call_to_param(func_name, &mut arguments); - return ControlFlow::Break(param); + let arguments = args.clone(); + return ControlFlow::Break((func_name.to_string(), arguments)); } _ => (), } @@ -567,10 +625,28 @@ impl VisitorMut for BadFunctionRemover { } } -fn remove_invalid_function_calls(stmt: &mut Statement, params: &mut Vec) { - let mut remover = BadFunctionRemover; - if let ControlFlow::Break(param) = stmt.visit(&mut remover) { - params.push(param); +fn validate_function_calls(stmt: &Statement) -> anyhow::Result<()> { + let mut finder = InvalidFunctionFinder; + if let ControlFlow::Break((func_name, args)) = stmt.visit(&mut finder) { + let args_str = FormatArguments(&args); + let error_msg = format!( + "Invalid SQLPage function call: sqlpage.{func_name}({args_str})\n\n\ + Arbitrary SQL expressions as function arguments are not supported.\n\n\ + SQLPage functions can either:\n\ + 1. Run BEFORE the query (to provide input values)\n\ + 2. Run AFTER the query (to process the results)\n\ + But they can't run DURING the query - the database doesn't know how to call them!\n\n\ + To fix this, you can either:\n\ + 1. Store the function argument in a variable first:\n\ + SET {func_name}_arg = ...;\n\ + SET {func_name}_result = sqlpage.{func_name}(${func_name}_arg);\n\ + SELECT * FROM example WHERE xxx = ${func_name}_result;\n\n\ + 2. Or move the function to the top level to process results:\n\ + SELECT sqlpage.{func_name}(...) FROM example;" + ); + Err(anyhow::anyhow!(error_msg)) + } else { + Ok(()) } } @@ -724,14 +800,15 @@ fn function_arg_expr(arg: &mut FunctionArg) -> Option<&mut Expr> { #[inline] #[must_use] -pub fn make_placeholder(db_kind: AnyKind, arg_number: usize) -> String { - if let Some((_, prefix)) = PLACEHOLDER_PREFIXES - .iter() - .find(|(kind, _)| *kind == db_kind) +pub fn make_tmp_placeholder(db_kind: AnyKind, arg_number: usize) -> String { + let prefix = if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = + DB_PLACEHOLDERS.iter().find(|(kind, _)| *kind == db_kind) { - return format!("{prefix}{arg_number}"); - } - DEFAULT_PLACEHOLDER.to_string() + prefix + } else { + TEMP_PLACEHOLDER_PREFIX + }; + format!("{prefix}{arg_number}") } fn extract_ident_param(Ident { value, .. }: &mut Ident) -> Option { @@ -749,17 +826,14 @@ impl VisitorMut for ParameterExtractor { match value { Expr::Identifier(ident) => { if let Some(param) = extract_ident_param(ident) { - *value = self.make_placeholder(); - self.parameters.push(param); + self.replace_with_placeholder(value, param); } } Expr::Value(Value::Placeholder(param)) if !self.is_own_placeholder(param) => // this check is to avoid recursively replacing placeholders in the form of '?', or '$1', '$2', which we emit ourselves { - let new_expr = self.make_placeholder(); let name = std::mem::take(param); - self.parameters.push(map_param(name)); - *value = new_expr; + self.replace_with_placeholder(value, map_param(name)); } Expr::Function(Function { name: ObjectName(func_name_parts), @@ -776,8 +850,9 @@ impl VisitorMut for ParameterExtractor { }) if is_sqlpage_func(func_name_parts) && are_params_extractable(args) => { let func_name = sqlpage_func_name(func_name_parts); log::trace!("Handling builtin function: {func_name}"); - let arguments = std::mem::take(args); - *value = self.handle_builtin_function(func_name, arguments); + let mut arguments = std::mem::take(args); + let param = func_call_to_param(func_name, &mut arguments); + self.replace_with_placeholder(value, param); } // Replace 'str1' || 'str2' with CONCAT('str1', 'str2') for MSSQL Expr::BinaryOp { @@ -980,15 +1055,16 @@ mod test { let mut ast = parse_postgres_stmt("select $a from t where $x > $a OR $x = sqlpage.cookie('cookoo')"); let parameters = ParameterExtractor::extract_parameters(&mut ast, AnyKind::Postgres); + // $a -> $1 + // $x -> $2 + // sqlpage.cookie(...) -> $3 assert_eq!( ast.to_string(), - "SELECT CAST($1 AS TEXT) FROM t WHERE CAST($2 AS TEXT) > CAST($3 AS TEXT) OR CAST($4 AS TEXT) = CAST($5 AS TEXT)" + "SELECT CAST($1 AS TEXT) FROM t WHERE CAST($2 AS TEXT) > CAST($1 AS TEXT) OR CAST($2 AS TEXT) = CAST($3 AS TEXT)" ); assert_eq!( parameters, [ - StmtParam::PostOrGet("a".to_string()), - StmtParam::PostOrGet("x".to_string()), StmtParam::PostOrGet("a".to_string()), StmtParam::PostOrGet("x".to_string()), StmtParam::FunctionCall(SqlPageFunctionCall { @@ -1005,7 +1081,7 @@ mod test { let parameters = ParameterExtractor::extract_parameters(&mut ast, AnyKind::Sqlite); assert_eq!( ast.to_string(), - "SELECT CAST(? AS TEXT), CAST(? AS TEXT) FROM t" + "SELECT CAST(?1 AS TEXT), CAST(?2 AS TEXT) FROM t" ); assert_eq!( parameters, @@ -1148,7 +1224,7 @@ mod test { assert!(ParameterExtractor { db_kind: AnyKind::Postgres, - parameters: vec![StmtParam::Get('x'.to_string())] + parameters: vec![StmtParam::Get("x".to_string())] } .is_own_placeholder("$2")); @@ -1162,7 +1238,7 @@ mod test { db_kind: AnyKind::Sqlite, parameters: vec![] } - .is_own_placeholder("?")); + .is_own_placeholder("?1")); assert!(!ParameterExtractor { db_kind: AnyKind::Sqlite, @@ -1332,18 +1408,18 @@ mod test { json_array(1, 2, 3) AS json_col2, (SELECT json_build_object('nested', subq.val) FROM (SELECT AVG(x) AS val FROM generate_series(1, 5) x) subq - ) AS json_col3, -- not supported because of the subquery - CASE - WHEN EXISTS (SELECT 1 FROM json_cte WHERE cte_json->>'a' = '2') - THEN to_json(ARRAY(SELECT cte_json FROM json_cte)) - ELSE json_build_array() - END AS json_col4, -- not supported because of the CASE - json_unknown_fn(regular_column) AS non_json_col, - CAST(json_col1 AS json) AS json_col6 - FROM some_table - CROSS JOIN json_cte - WHERE json_typeof(json_col1) = 'object' - "; + ) AS json_col3, -- not supported because of the subquery + CASE + WHEN EXISTS (SELECT 1 FROM json_cte WHERE cte_json->>'a' = '2') + THEN to_json(ARRAY(SELECT cte_json FROM json_cte)) + ELSE json_build_array() + END AS json_col4, -- not supported because of the CASE + json_unknown_fn(regular_column) AS non_json_col, + CAST(json_col1 AS json) AS json_col6 + FROM some_table + CROSS JOIN json_cte + WHERE json_typeof(json_col1) = 'object' + "; let stmt = parse_postgres_stmt(sql); let json_columns = extract_json_columns(&stmt, AnyKind::Sqlite); @@ -1412,4 +1488,92 @@ mod test { assert!(json_columns.contains(&"item".to_string())); assert!(!json_columns.contains(&"title".to_string())); } + + #[test] + fn test_positional_placeholders() { + let sql = "select \ + @SQLPAGE_TEMP10 as a1, \ + @SQLPAGE_TEMP9 as a2, \ + @SQLPAGE_TEMP8 as a3, \ + @SQLPAGE_TEMP7 as a4, \ + @SQLPAGE_TEMP6 as a5, \ + @SQLPAGE_TEMP5 as a6, \ + @SQLPAGE_TEMP4 as a7, \ + @SQLPAGE_TEMP3 as a8, \ + @SQLPAGE_TEMP2 as a9, \ + @SQLPAGE_TEMP1 as a10 \ + @SQLPAGE_TEMP10 as a1bis \ + from t"; + let mut stmt = StmtWithParams { + query: sql.to_string(), + query_position: SourceSpan { + start: SourceLocation { line: 1, column: 1 }, + end: SourceLocation { line: 1, column: 1 }, + }, + params: vec![ + StmtParam::PostOrGet("x1".to_string()), + StmtParam::PostOrGet("x2".to_string()), + StmtParam::PostOrGet("x3".to_string()), + StmtParam::PostOrGet("x4".to_string()), + StmtParam::PostOrGet("x5".to_string()), + StmtParam::PostOrGet("x6".to_string()), + StmtParam::PostOrGet("x7".to_string()), + StmtParam::PostOrGet("x8".to_string()), + StmtParam::PostOrGet("x9".to_string()), + StmtParam::PostOrGet("x10".to_string()), + ], + delayed_functions: vec![], + json_columns: vec![], + }; + transform_to_positional_placeholders(&mut stmt, AnyKind::MySql); + assert_eq!( + stmt.query, + "select \ + ? as a1, \ + ? as a2, \ + ? as a3, \ + ? as a4, \ + ? as a5, \ + ? as a6, \ + ? as a7, \ + ? as a8, \ + ? as a9, \ + ? as a10 \ + ? as a1bis \ + from t" + ); + assert_eq!( + stmt.params, + vec![ + StmtParam::PostOrGet("x10".to_string()), + StmtParam::PostOrGet("x9".to_string()), + StmtParam::PostOrGet("x8".to_string()), + StmtParam::PostOrGet("x7".to_string()), + StmtParam::PostOrGet("x6".to_string()), + StmtParam::PostOrGet("x5".to_string()), + StmtParam::PostOrGet("x4".to_string()), + StmtParam::PostOrGet("x3".to_string()), + StmtParam::PostOrGet("x2".to_string()), + StmtParam::PostOrGet("x1".to_string()), + StmtParam::PostOrGet("x10".to_string()), + ] + ); + } + + #[test] + fn test_set_variable_error_handling() { + let sql = "set x = db_function(sqlpage.fetch(other_db_function()))"; + for &(dialect, db_kind) in ALL_DIALECTS { + let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); + let stmt = parse_single_statement(&mut parser, db_kind, sql); + if let Some(ParsedStatement::Error(err)) = stmt { + assert!( + err.to_string().contains("Invalid SQLPage function call"), + "Expected error for invalid function, got: {err}" + ); + } else { + panic!("Expected error for invalid function, got: {stmt:#?}"); + } + } + } } diff --git a/tests/sql_test_files/it_works_case_variables.sql b/tests/sql_test_files/it_works_case_variables.sql new file mode 100644 index 00000000..44d68510 --- /dev/null +++ b/tests/sql_test_files/it_works_case_variables.sql @@ -0,0 +1,10 @@ +-- https://github.com/sqlpage/SQLPage/issues/818 + +set success = 'It works !'; +set failure = 'You should never see this'; + +select 'text' as component, + case $success + when $success then $success + when $failure then $failure + end AS contents; \ No newline at end of file From a10a77794dfecc278bd9de1f3c014caa63b011f0 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sat, 22 Feb 2025 14:36:29 +0100 Subject: [PATCH 2/9] fix navbar muted color fixes https://github.com/sqlpage/SQLPage/issues/822 --- CHANGELOG.md | 1 + sqlpage/sqlpage.css | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7d28f00..66abbb25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Implement parameter deduplication. Now, when you write `select $x where $x is not null`, the value of `$x` is sent to the database only once. It used to be sent as many times as `$x` appeared in the statement. - Improve error messages on invalid sqlpage function calls. The messages now contain actionable advice. +- Fix top navigation bar links color. They appeared "muted", with low contrast, since v0.33 ## 0.33.0 (2025-02-15) diff --git a/sqlpage/sqlpage.css b/sqlpage/sqlpage.css index 50eda661..21b4b82f 100644 --- a/sqlpage/sqlpage.css +++ b/sqlpage/sqlpage.css @@ -8,6 +8,11 @@ --tblr-code-bg: #e4f1ff; } +.navbar { + /* https://github.com/sqlpage/SQLPage/issues/822 */ + --tblr-navbar-color: rgba(var(--tblr-body-color-rgb), 0.8); +} + [data-bs-theme="dark"] .alert:not(.alert-important) { /* See https://github.com/tabler/tabler/issues/1607 */ background-color: var(--tblr-bg-surface); From b11136a5911f22bdc403b41d79cbd60f4c8dd253 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sat, 22 Feb 2025 14:47:20 +0100 Subject: [PATCH 3/9] update dependencies fixes https://github.com/sqlpage/SQLPage/issues/797 --- CHANGELOG.md | 1 + Cargo.lock | 42 +++++++++++++++++++++--------------------- Cargo.toml | 2 +- sqlpage/apexcharts.js | 2 +- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66abbb25..01b1c6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Now, when you write `select $x where $x is not null`, the value of `$x` is sent to the database only once. It used to be sent as many times as `$x` appeared in the statement. - Improve error messages on invalid sqlpage function calls. The messages now contain actionable advice. - Fix top navigation bar links color. They appeared "muted", with low contrast, since v0.33 +- update to apex charts v4.5.0. This fixes a bug where tick positions in scatter plots would be incorrect. ## 0.33.0 (2025-02-15) diff --git a/Cargo.lock b/Cargo.lock index 4afd3430..48efda03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -799,9 +799,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.14" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ "jobserver", "libc", @@ -2145,9 +2145,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "markdown" @@ -2199,9 +2199,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", ] @@ -2412,9 +2412,9 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pem" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ "base64 0.22.1", "serde", @@ -2673,9 +2673,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" dependencies = [ "bitflags", ] @@ -2728,9 +2728,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.9" +version = "0.17.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" +checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839" dependencies = [ "cc", "cfg-if", @@ -3127,7 +3127,7 @@ dependencies = [ [[package]] name = "sqlpage" -version = "0.33.0" +version = "0.33.1" dependencies = [ "actix-multipart", "actix-rt", @@ -3764,9 +3764,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" +checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1" [[package]] name = "vcpkg" @@ -4251,27 +4251,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.14+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 1d855767..d066713c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sqlpage" -version = "0.33.0" +version = "0.33.1" edition = "2021" description = "Build data user interfaces entirely in SQL. A web server that takes .sql files and formats the query result using pre-made configurable professional-looking components." keywords = ["web", "sql", "framework"] diff --git a/sqlpage/apexcharts.js b/sqlpage/apexcharts.js index 99ad002c..9ab85033 100644 --- a/sqlpage/apexcharts.js +++ b/sqlpage/apexcharts.js @@ -1,4 +1,4 @@ -/* !include https://cdn.jsdelivr.net/npm/apexcharts@4.4.0/dist/apexcharts.min.js */ +/* !include https://cdn.jsdelivr.net/npm/apexcharts@4.5.0/dist/apexcharts.min.js */ sqlpage_chart = (() => { function sqlpage_chart() { From 5fd1c83df15b9a7f8541131951da07185f226fea Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Mon, 24 Feb 2025 22:01:30 +0100 Subject: [PATCH 4/9] new function: fetch_with_meta (#827) * new function: fetch_with_meta closes https://github.com/sqlpage/SQLPage/issues/792 * fmt * clippy auto * test fetch_with_meta * fix tests * update deps * retry failed deps downloads * better json serializing * test fetch_with_meta error handling * add logging * fix tests closes #792 --- Cargo.lock | 32 ++-- build.rs | 45 ++++-- .../sqlpage/migrations/40_fetch.sql | 6 + .../sqlpage/migrations/58_fetch_with_meta.sql | 86 ++++++++++ .../database/sqlpage_functions/functions.rs | 148 +++++++++++++++--- tests/index.rs | 7 +- .../it_works_fetch_with_meta_error.sql | 7 + .../it_works_fetch_with_meta_simple.sql | 13 ++ 8 files changed, 289 insertions(+), 55 deletions(-) create mode 100644 examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql create mode 100644 tests/sql_test_files/it_works_fetch_with_meta_error.sql create mode 100644 tests/sql_test_files/it_works_fetch_with_meta_simple.sql diff --git a/Cargo.lock b/Cargo.lock index 48efda03..e5053a03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,9 +830,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" dependencies = [ "clap_builder", "clap_derive", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" dependencies = [ "anstream", "anstyle", @@ -1254,9 +1254,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "encoding_rs" @@ -1346,9 +1346,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -2049,9 +2049,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libflate" @@ -2616,7 +2616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", + "rand_core 0.9.2", "zerocopy 0.8.20", ] @@ -2637,7 +2637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.2", ] [[package]] @@ -2651,9 +2651,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "7a509b1a2ffbe92afab0e55c8fd99dea1c280e8171bd2d88682bb20bc41cbc2c" dependencies = [ "getrandom 0.3.1", "zerocopy 0.8.20", @@ -2728,9 +2728,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" dependencies = [ "cc", "cfg-if", diff --git a/build.rs b/build.rs index 3c2bdeea..8c3ae77f 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,5 @@ use actix_rt::spawn; +use actix_rt::time::sleep; use libflate::gzip; use std::collections::hash_map::DefaultHasher; use std::fs::File; @@ -99,21 +100,37 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write) } async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) { - let mut resp = client.get(url).send().await.unwrap_or_else(|err| { - let path = make_url_path(url); - panic!( - "We need to download external frontend dependencies to build the static frontend. \n\ - Could not download static asset. You can manually download the file with: \n\ - curl {url:?} > {path:?} \n\ - {err}" - ) - }); - if resp.status() != 200 { - panic!("Received {} status code from {}", resp.status(), url); + let mut attempt = 1; + let max_attempts = 2; + + loop { + match client.get(url).send().await { + Ok(mut resp) => { + if resp.status() != 200 { + panic!("Received {} status code from {}", resp.status(), url); + } + let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); + std::fs::write(path, &bytes) + .expect("Failed to write external frontend dependency to local file"); + break; + } + Err(err) => { + if attempt >= max_attempts { + let path = make_url_path(url); + panic!( + "We need to download external frontend dependencies to build the static frontend. \n\ + Could not download static asset after {} attempts. You can manually download the file with: \n\ + curl {url:?} > {path:?} \n\ + {err}", + max_attempts + ); + } + sleep(Duration::from_secs(1)).await; + println!("cargo:warning=Retrying download of {url} after {err}."); + attempt += 1; + } + } } - let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); - std::fs::write(path, &bytes) - .expect("Failed to write external frontend dependency to local file"); } // Given a filename, creates a new unique filename based on the file contents diff --git a/examples/official-site/sqlpage/migrations/40_fetch.sql b/examples/official-site/sqlpage/migrations/40_fetch.sql index cad1af2e..002bc0c9 100644 --- a/examples/official-site/sqlpage/migrations/40_fetch.sql +++ b/examples/official-site/sqlpage/migrations/40_fetch.sql @@ -88,6 +88,12 @@ The fetch function accepts either a URL string, or a JSON object with the follow - `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0. - `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0. +# Error handling and reading response headers + +If the request fails, this function throws an error, that will be displayed to the user. +The response headers are not available for inspection. + +If you need to handle errors or inspect the response headers, use [`sqlpage.fetch_with_meta`](?function=fetch_with_meta). ' ); INSERT INTO sqlpage_function_parameters ( diff --git a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql new file mode 100644 index 00000000..296071d2 --- /dev/null +++ b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql @@ -0,0 +1,86 @@ +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" + ) +VALUES ( + 'fetch_with_meta', + '0.34.0', + 'transfer-vertical', + 'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body. + +This function is similar to [`fetch`](?function=fetch), but returns a JSON object containing detailed information about the response. +The returned object has the following structure: +```json +{ + "status": 200, + "headers": { + "content-type": "text/html", + "content-length": "1234" + }, + "body": "a string, or a json object, depending on the content type", + "error": "error message if any" +} +``` + +If the request fails or encounters an error (e.g., network issues, invalid UTF-8 response), instead of throwing an error, +the function returns a JSON object with an "error" field containing the error message. + +### Example: Basic Usage + +```sql +-- Make a request and get detailed response information +set response = sqlpage.fetch_with_meta(''https://pokeapi.co/api/v2/pokemon/ditto''); + +-- redirect the user to an error page if the request failed +select ''redirect'' as component, ''error.sql'' as url +where + json_extract($response, ''$.error'') is not null + or json_extract($response, ''$.status'') != 200; + +-- Extract data from the response json body +select ''card'' as component; +select + json_extract($response, ''$.body.name'') as title, + json_extract($response, ''$.body.abilities[0].ability.name'') as description +from $response; +``` + +### Example: Advanced Request with Authentication + +```sql +set request = json_object( + ''method'', ''POST'', + ''url'', ''https://sqlpage.free.beeceptor.com'', + ''headers'', json_object( + ''Content-Type'', ''application/json'', + ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'') + ), + ''body'', json_object( + ''key'', ''value'' + ) +); +set response = sqlpage.fetch_with_meta($request); + +-- Check response content type +select ''debug'' as component, $response as response; +``` + +The function accepts the same parameters as the [`fetch` function](?function=fetch).' + ); + +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" + ) +VALUES ( + 'fetch_with_meta', + 1, + 'url', + 'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.', + 'TEXT' + ); \ No newline at end of file diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index 0f32f4cb..1b52f706 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -24,6 +24,7 @@ super::function_definition_macro::sqlpage_functions! { exec((&RequestInfo), program_name: Cow, args: Vec>); fetch((&RequestInfo), http_request: SqlPageFunctionParam>); + fetch_with_meta((&RequestInfo), http_request: SqlPageFunctionParam>); hash_password(password: Option); header((&RequestInfo), name: Cow); @@ -135,16 +136,13 @@ async fn exec<'a>( Ok(String::from_utf8_lossy(&res.stdout).into_owned()) } -async fn fetch( - request: &RequestInfo, - http_request: super::http_fetch_request::HttpFetchRequest<'_>, -) -> anyhow::Result { +fn build_request<'a>( + client: &'a awc::Client, + http_request: &'a super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { use awc::http::Method; - let client = make_http_client(&request.app_state.config) - .with_context(|| "Unable to create an HTTP client")?; - - let method = if let Some(method) = http_request.method { - Method::from_str(&method).with_context(|| format!("Invalid HTTP method: {method}"))? + let method = if let Some(method) = &http_request.method { + Method::from_str(method).with_context(|| format!("Invalid HTTP method: {method}"))? } else { Method::GET }; @@ -152,36 +150,56 @@ async fn fetch( if let Some(timeout) = http_request.timeout_ms { req = req.timeout(core::time::Duration::from_millis(timeout)); } - for (k, v) in http_request.headers { + for (k, v) in &http_request.headers { req = req.insert_header((k.as_ref(), v.as_ref())); } - if let Some(username) = http_request.username { - let password = http_request.password.unwrap_or_default(); + if let Some(username) = &http_request.username { + let password = http_request.password.as_deref().unwrap_or_default(); req = req.basic_auth(username, password); } + Ok(req) +} + +fn prepare_request_body( + body: &serde_json::value::RawValue, + mut req: awc::ClientRequest, +) -> anyhow::Result<(String, awc::ClientRequest)> { + let val = body.get(); + let body_str = if val.starts_with('"') { + serde_json::from_str::<'_, String>(val).with_context(|| { + format!("Invalid JSON string in the body of the HTTP request: {val}") + })? + } else { + req = req.content_type("application/json"); + val.to_owned() + }; + Ok((body_str, req)) +} + +async fn fetch( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + log::info!("Fetching {}", http_request.url); - let mut response = if let Some(body) = http_request.body { - let val = body.get(); - // The body can be either json, or a string representing a raw body - let body = if val.starts_with('"') { - serde_json::from_str::<'_, String>(val).with_context(|| { - format!("Invalid JSON string in the body of the HTTP request: {val}") - })? - } else { - req = req.content_type("application/json"); - val.to_owned() - }; + let mut response = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; req.send_body(body) } else { req.send() } .await .map_err(|e| anyhow!("Unable to fetch {}: {e}", http_request.url))?; + log::debug!( "Finished fetching {}. Status: {}", http_request.url, response.status() ); + let body = response .body() .await @@ -199,6 +217,90 @@ async fn fetch( Ok(response_str) } +async fn fetch_with_meta( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + use serde::{ser::SerializeMap, Serializer}; + + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + + log::info!("Fetching {} with metadata", http_request.url); + let response_result = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; + req.send_body(body).await + } else { + req.send().await + }; + + let mut resp_str = Vec::new(); + let mut encoder = serde_json::Serializer::new(&mut resp_str); + let mut obj = encoder.serialize_map(Some(3))?; + match response_result { + Ok(mut response) => { + obj.serialize_entry("status", &response.status().as_u16())?; + + let headers = response.headers(); + + let is_json = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .starts_with("application/json"); + + obj.serialize_entry( + "headers", + &headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default())) + .collect::>(), + )?; + + match response.body().await { + Ok(body) => { + let body_bytes = body.to_vec(); + let body_str = String::from_utf8(body_bytes); + + match body_str { + Ok(body_str) if is_json => { + obj.serialize_entry( + "body", + &serde_json::value::RawValue::from_string(body_str)?, + )?; + } + Ok(body_str) => { + obj.serialize_entry("body", &body_str)?; + } + Err(utf8_err) => { + let mut base64_string = String::new(); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + utf8_err.as_bytes(), + &mut base64_string, + ); + obj.serialize_entry("body", &base64_string)?; + } + } + } + Err(e) => { + log::warn!("Failed to read response body: {e}"); + obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?; + } + } + } + Err(e) => { + log::warn!("Request failed: {e}"); + obj.serialize_entry("error", &format!("Request failed: {e}"))?; + } + } + + obj.end()?; + let return_value = String::from_utf8(resp_str)?; + Ok(return_value) +} + static NATIVE_CERTS: OnceLock> = OnceLock::new(); fn make_http_client(config: &crate::app_config::AppConfig) -> anyhow::Result { diff --git a/tests/index.rs b/tests/index.rs index 590eb520..1f1dee8b 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -136,7 +136,10 @@ fn start_echo_server() -> ServerHandle { } f.push(b'|'); f.extend_from_slice(&r.extract::().await?); - let resp = HttpResponse::Ok().body(f); + let resp = HttpResponse::Ok() + .insert_header((header::DATE, "Mon, 24 Feb 2025 12:00:00 GMT")) + .insert_header((header::CONTENT_TYPE, "text/plain")) + .body(f); Ok(r.into_response(resp)) } let server = actix_web::HttpServer::new(move || { @@ -201,7 +204,7 @@ async fn test_files() { ); assert!( !lowercase_body.contains("error"), - "{body}\nexpected to not contain: error" + "{req_str}\n{body}\nexpected to not contain: error" ); } else if stem.starts_with("error_") { let rest = stem.strip_prefix("error_").unwrap(); diff --git a/tests/sql_test_files/it_works_fetch_with_meta_error.sql b/tests/sql_test_files/it_works_fetch_with_meta_error.sql new file mode 100644 index 00000000..4c6d2c88 --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_error.sql @@ -0,0 +1,7 @@ +set res = sqlpage.fetch_with_meta('http://not-a-real-url'); + +select 'text' as component, + case + when json_extract($res, '$.error') LIKE '%Request failed%' then 'It works !' + else CONCAT('Error! Got: ', $res) + end as contents; \ No newline at end of file diff --git a/tests/sql_test_files/it_works_fetch_with_meta_simple.sql b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql new file mode 100644 index 00000000..d4d8bfaa --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql @@ -0,0 +1,13 @@ +set res = sqlpage.fetch_with_meta('{ + "method": "PUT", + "url": "http://localhost:62802/hello_world", + "headers": { + "user-agent": "myself" + } +}'); + +select 'text' as component, + case + when $res LIKE '%"status":200%' AND $res LIKE '%"headers":{%' AND $res LIKE '%"body":"%' then 'It works !' + else 'Error! Got: ' || $res + end as contents; \ No newline at end of file From e766a7fb149f6f7979d1c42da8d0828fb0929e43 Mon Sep 17 00:00:00 2001 From: Kuro <71044351+Kitsune-Kuro@users.noreply.github.com> Date: Mon, 24 Feb 2025 22:10:41 +0100 Subject: [PATCH 5/9] Insert the newest sqlpage version (#826) Co-authored-by: Kuro --- .../official-site/your-first-sql-website/index.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/official-site/your-first-sql-website/index.sql b/examples/official-site/your-first-sql-website/index.sql index df966f23..c50644d8 100644 --- a/examples/official-site/your-first-sql-website/index.sql +++ b/examples/official-site/your-first-sql-website/index.sql @@ -1,3 +1,7 @@ +SET url = 'https://api.github.com/repos/sqlpage/SQLPage/releases/latest'; +SET api_results = sqlpage.fetch($url); +SET sqlpage_version = json_extract($api_results, '$.tag_name'); + select 'http_header' as component, 'public, max-age=300, stale-while-revalidate=3600, stale-if-error=86400' as "Cache-Control", '; rel="canonical"' as "Link"; @@ -36,10 +40,10 @@ Let''s create a simple website with a database from scratch, to learn SQLPage ba ELSE 'https://github.com/sqlpage/SQLPage/releases' END AS link, CASE $os - WHEN 'macos' THEN 'Install SQLPage using Homebrew' - WHEN 'windows' THEN 'Download SQLPage for Windows' - WHEN 'linux' THEN 'Download SQLPage for Linux' - ELSE 'Download SQLPage' + WHEN 'macos' THEN CONCAT('Install SQLPage ', $sqlpage_version, ' using Homebrew') + WHEN 'windows' THEN CONCAT('Download SQLPage ', $sqlpage_version, ' for Windows') + WHEN 'linux' THEN CONCAT('Download SQLPage ', $sqlpage_version, ' for Linux') + ELSE CONCAT('Download SQLPage ', $sqlpage_version) END AS link_text; SELECT 'alert' as component, From 433162a9f29da35278e5f9dd73665ebec25193a7 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 22:09:54 +0100 Subject: [PATCH 6/9] error handling --- CHANGELOG.md | 7 +++++++ .../database/sqlpage_functions/functions.rs | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01b1c6a5..c4a50592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ - Improve error messages on invalid sqlpage function calls. The messages now contain actionable advice. - Fix top navigation bar links color. They appeared "muted", with low contrast, since v0.33 - update to apex charts v4.5.0. This fixes a bug where tick positions in scatter plots would be incorrect. +- New function: `sqlpage.fetch_with_meta` + - This function is similar to `sqlpage.fetch`, but it returns a json object with the following properties: + - `status`: the http status code of the response. + - `headers`: a json object with the response headers. + - `body`: the response body. + - `error`: an error message if the request failed. + - This is useful when interacting with complex or unreliable external APIs. ## 0.33.0 (2025-02-15) diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index 1b52f706..6abe85d6 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -240,7 +240,13 @@ async fn fetch_with_meta( let mut obj = encoder.serialize_map(Some(3))?; match response_result { Ok(mut response) => { - obj.serialize_entry("status", &response.status().as_u16())?; + let status = response.status(); + obj.serialize_entry("status", &status.as_u16())?; + let mut has_error = false; + if status.is_server_error() { + has_error = true; + obj.serialize_entry("error", &format!("Server error: {status}"))?; + } let headers = response.headers(); @@ -286,7 +292,12 @@ async fn fetch_with_meta( } Err(e) => { log::warn!("Failed to read response body: {e}"); - obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?; + if !has_error { + obj.serialize_entry( + "error", + &format!("Failed to read response body: {e}"), + )?; + } } } } From 6785ab9c0a1b53c37f22f3d331850d07b05c04a3 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 22:16:26 +0100 Subject: [PATCH 7/9] fail silently when fetching version is impossible --- .../official-site/your-first-sql-website/index.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/official-site/your-first-sql-website/index.sql b/examples/official-site/your-first-sql-website/index.sql index c50644d8..fe380cdc 100644 --- a/examples/official-site/your-first-sql-website/index.sql +++ b/examples/official-site/your-first-sql-website/index.sql @@ -1,7 +1,3 @@ -SET url = 'https://api.github.com/repos/sqlpage/SQLPage/releases/latest'; -SET api_results = sqlpage.fetch($url); -SET sqlpage_version = json_extract($api_results, '$.tag_name'); - select 'http_header' as component, 'public, max-age=300, stale-while-revalidate=3600, stale-if-error=86400' as "Cache-Control", '; rel="canonical"' as "Link"; @@ -22,6 +18,13 @@ select 'dynamic' as component, json_patch(json_extract(properties, '$[0]'), json )) as properties FROM example WHERE component = 'shell' LIMIT 1; +SET req = '{ + "url": "https://api.github.com/repos/sqlpage/SQLPage/releases/latest", + "timeout_ms": 200 +}'; +SET api_results = sqlpage.fetch_with_meta($req); +SET sqlpage_version = COALESCE(json_extract($api_results, '$.body.tag_name'), ''); + SELECT 'hero' as component, 'Your first SQL Website' as title, '[SQLPage](/) is a free tool for building data-driven apps quickly. From f3be3854b619dc892d810f134ad994b3deffb34a Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 22:18:26 +0100 Subject: [PATCH 8/9] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a50592..92c30420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG.md -## 0.33.1 (unreleased) +## 0.33.1 (2025-02-25) - Fix a bug where the table component would not format numbers if sorting was not enabled. - Fix a bug with date sorting in the table component. From e75fc22d7f3287cacd16fcfeef8f5959e81b91f2 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 22:22:03 +0100 Subject: [PATCH 9/9] fix tests on mssql no json_extract --- tests/sql_test_files/it_works_fetch_with_meta_error.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sql_test_files/it_works_fetch_with_meta_error.sql b/tests/sql_test_files/it_works_fetch_with_meta_error.sql index 4c6d2c88..721797fb 100644 --- a/tests/sql_test_files/it_works_fetch_with_meta_error.sql +++ b/tests/sql_test_files/it_works_fetch_with_meta_error.sql @@ -2,6 +2,6 @@ set res = sqlpage.fetch_with_meta('http://not-a-real-url'); select 'text' as component, case - when json_extract($res, '$.error') LIKE '%Request failed%' then 'It works !' + when $res LIKE '%"error":"Request failed%' then 'It works !' else CONCAT('Error! Got: ', $res) end as contents; \ No newline at end of file