Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Normalizes and sanitizes UTF-8 input #7102

Merged
merged 11 commits into from
Oct 7, 2021
Merged

Normalizes and sanitizes UTF-8 input #7102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f693da3
Normalizes Unicode in preparsecode()
Sesquipedalian Sep 13, 2021
5a4dbd2
Normalize Unicode in $smcFunc['htmlspecialchars'] & $smcFunc['strto*']
Sesquipedalian Sep 14, 2021
cdbb309
Normalizes Unicode in strings for censorText
Sesquipedalian Sep 17, 2021
4e8f5ca
Normalizes Unicode when editing strings via the built in language editor
Sesquipedalian Sep 17, 2021
9573b70
Normalizes Unicode in input for profile fields
Sesquipedalian Sep 17, 2021
17d7222
Normalizes Unicode in various input fields
Sesquipedalian Sep 17, 2021
3a73a1b
Improves character sanitization of usernames
Sesquipedalian Sep 17, 2021
3e0692e
Adds utf8_sanitize_invisibles()
Sesquipedalian Sep 27, 2021
1b98602
Improves input sanitization in preparsecode() and profile fields
Sesquipedalian Sep 28, 2021
8569fbd
Adds missing globals
Sesquipedalian Sep 29, 2021
0a5a92e
Minor fixes sanitizing join control chars
Sesquipedalian Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions Sources/Load.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,10 @@ function reloadSettings()
$num = $string[0] === 'x' ? hexdec(substr($string, 1)) : (int) $string;
return $num < 0x20 || $num > 0x10FFFF || ($num >= 0xD800 && $num <= 0xDFFF) || $num === 0x202E || $num === 0x202D ? '' : '&#' . $num . ';';
},
'htmlspecialchars' => function($string, $quote_style = ENT_COMPAT, $charset = 'ISO-8859-1') use ($ent_check, $utf8, $fix_utf8mb4)
'htmlspecialchars' => function($string, $quote_style = ENT_COMPAT, $charset = 'ISO-8859-1') use ($ent_check, $utf8, $fix_utf8mb4, &$smcFunc)
{
$string = $smcFunc['normalize']($string);

return $fix_utf8mb4($ent_check(htmlspecialchars($string, $quote_style, $utf8 ? 'UTF-8' : $charset)));
},
'htmltrim' => function($string) use ($utf8, $ent_check)
Expand Down Expand Up @@ -206,8 +208,10 @@ function reloadSettings()
$ent_arr = preg_split('~(' . $ent_list . '|.)~' . ($utf8 ? 'u' : '') . '', $ent_check($string), -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
return $length === null ? implode('', array_slice($ent_arr, $start)) : implode('', array_slice($ent_arr, $start, $length));
},
'strtolower' => $utf8 ? function($string) use ($sourcedir)
'strtolower' => $utf8 ? function($string) use ($sourcedir, &$smcFunc)
{
$string = $smcFunc['normalize']($string);

if (!function_exists('mb_strtolower'))
{
require_once($sourcedir . '/Subs-Charset.php');
Expand All @@ -216,9 +220,9 @@ function reloadSettings()

return mb_strtolower($string, 'UTF-8');
} : 'strtolower',
'strtoupper' => $utf8 ? function($string)
'strtoupper' => $utf8 ? function($string) use ($sourcedir, &$smcFunc)
{
global $sourcedir;
$string = $smcFunc['normalize']($string);

if (!function_exists('mb_strtolower'))
{
Expand Down
2 changes: 1 addition & 1 deletion Sources/ManageCalendar.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ function EditHoliday()
validateToken('admin-eh');

// Not too long good sir?
$_REQUEST['title'] = $smcFunc['substr']($_REQUEST['title'], 0, 60);
$_REQUEST['title'] = $smcFunc['substr']($smcFunc['normalize']($_REQUEST['title']), 0, 60);
$_REQUEST['holiday'] = isset($_REQUEST['holiday']) ? (int) $_REQUEST['holiday'] : 0;

if (isset($_REQUEST['delete']))
Expand Down
2 changes: 2 additions & 0 deletions Sources/ManageLanguages.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1659,6 +1659,8 @@ function cleanLangString($string, $to_display = true)
}
else
{
$string = $smcFunc['normalize']($string);

// Keep track of what we're doing...
$in_string = 0;
// This is for deciding whether to HTML a quote.
Expand Down
2 changes: 1 addition & 1 deletion Sources/ManageMembergroups.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,7 @@ function EditMembergroup()
'group_name' => $smcFunc['htmlspecialchars']($_POST['group_name']),
'online_color' => $_POST['online_color'],
'icons' => $_POST['icons'],
'group_desc' => $_POST['group_desc'],
'group_desc' => $smcFunc['normalize']($_POST['group_desc']),
'tfa_required' => $_POST['group_tfa_force'],
)
);
Expand Down
4 changes: 2 additions & 2 deletions Sources/ManagePosts.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ function SetCensor()

// Set the new arrays and settings in the database.
$updates = array(
'censor_vulgar' => implode("\n", $censored_vulgar),
'censor_proper' => implode("\n", $censored_proper),
'censor_vulgar' => $smcFunc['normalize'](implode("\n", $censored_vulgar)),
'censor_proper' => $smcFunc['normalize'](implode("\n", $censored_proper)),
'allow_no_censored' => empty($_POST['allow_no_censored']) ? '0' : '1',
'censorWholeWord' => empty($_POST['censorWholeWord']) ? '0' : '1',
'censorIgnoreCase' => empty($_POST['censorIgnoreCase']) ? '0' : '1',
Expand Down
12 changes: 8 additions & 4 deletions Sources/ManageRegistration.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ function AdminRegister()

foreach ($_POST as $key => $value)
if (!is_array($_POST[$key]))
$_POST[$key] = htmltrim__recursive(str_replace(array("\n", "\r"), '', $_POST[$key]));
$_POST[$key] = htmltrim__recursive(str_replace(array("\n", "\r"), '', $smcFunc['normalize']($_POST[$key])));

$regOptions = array(
'interface' => 'admin',
Expand Down Expand Up @@ -224,6 +224,8 @@ function EditAgreement()
checkSession();
validateToken('admin-rega');

$_POST['agreement'] = $smcFunc['normalize']($_POST['agreement']);

// Off it goes to the agreement file.
$to_write = str_replace("\r", '', $_POST['agreement']);
$bytes = file_put_contents($boarddir . '/agreement' . $context['current_agreement'] . '.txt', $to_write, LOCK_EX);
Expand Down Expand Up @@ -270,14 +272,16 @@ function EditAgreement()
*/
function SetReserved()
{
global $txt, $context, $modSettings;
global $txt, $context, $modSettings, $smcFunc;

// Submitting new reserved words.
if (!empty($_POST['save_reserved_names']))
{
checkSession();
validateToken('admin-regr');

$_POST['reserved'] = $smcFunc['normalize']($_POST['reserved']);

// Set all the options....
updateSettings(array(
'reserveWord' => (isset($_POST['matchword']) ? '1' : '0'),
Expand Down Expand Up @@ -315,7 +319,7 @@ function SetReserved()
*/
function ModifyRegistrationSettings($return_config = false)
{
global $txt, $context, $scripturl, $modSettings, $sourcedir;
global $txt, $context, $scripturl, $modSettings, $sourcedir, $smcFunc;
global $language, $boarddir;

// This is really quite wanting.
Expand Down Expand Up @@ -359,7 +363,7 @@ function ModifyRegistrationSettings($return_config = false)
fatal_lang_error('admin_setting_coppa_require_contact');

// Post needs to take into account line breaks.
$_POST['coppaPost'] = str_replace("\n", '<br>', empty($_POST['coppaPost']) ? '' : $_POST['coppaPost']);
$_POST['coppaPost'] = str_replace("\n", '<br>', empty($_POST['coppaPost']) ? '' : $smcFunc['normalize']($_POST['coppaPost']));

call_integration_hook('integrate_save_registration_settings');

Expand Down
3 changes: 3 additions & 0 deletions Sources/ManageSearchEngines.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,9 @@ function EditSpider()
checkSession();
validateToken('admin-ses');

foreach (array('spider_name', 'spider_agent') as $key)
$_POST[$key] = trim($smcFunc['normalize']($_POST[$key]));

$ips = array();
// Check the IP range is valid.
$ip_sets = explode(',', $_POST['spider_ip']);
Expand Down
10 changes: 9 additions & 1 deletion Sources/ManageServer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ function ModifySettings()
*/
function ModifyGeneralSettings($return_config = false)
{
global $scripturl, $context, $txt, $modSettings, $boardurl, $sourcedir;
global $scripturl, $context, $txt, $modSettings, $boardurl, $sourcedir, $smcFunc;

// If no cert, force_ssl must remain 0
require_once($sourcedir . '/Subs.php');
Expand Down Expand Up @@ -198,6 +198,12 @@ function ModifyGeneralSettings($return_config = false)
{
call_integration_hook('integrate_save_general_settings');

foreach ($config_vars as $config_var)
{
if ($config_var[3] == 'text' && !empty($_POST[$config_var[0]]))
$_POST[$config_var[0]] = $smcFunc['normalize']($_POST[$config_var[0]]);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Must declare the global $smcFunc.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Sesquipedalian See #7127

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, @jdarwood007. Will submit fix shortly.

}

// Are we saving the stat collection?
if (!empty($_POST['enable_sm_stats']) && empty($modSettings['sm_stats_key']))
{
Expand Down Expand Up @@ -532,6 +538,8 @@ function hideGlobalCookies()
{
call_integration_hook('integrate_save_cookie_settings');

$_POST['cookiename'] = $smcFunc['normalize']($_POST['cookiename']);

// Local and global do not play nicely together.
if (!empty($_POST['localCookies']) && empty($_POST['globalCookies']))
unset ($_POST['globalCookies']);
Expand Down
3 changes: 2 additions & 1 deletion Sources/ManageSettings.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1963,9 +1963,10 @@ function EditCustomProfiles()
$mask = isset($_POST['mask']) ? $_POST['mask'] : '';
if ($mask == 'regex' && isset($_POST['regex']))
$mask .= $_POST['regex'];
$mask = $smcFunc['normalize']($mask);

$field_length = isset($_POST['max_length']) ? (int) $_POST['max_length'] : 255;
$enclose = isset($_POST['enclose']) ? $_POST['enclose'] : '';
$enclose = isset($_POST['enclose']) ? $smcFunc['normalize']($_POST['enclose']) : '';
$placement = isset($_POST['placement']) ? (int) $_POST['placement'] : 0;

// Select options?
Expand Down
16 changes: 14 additions & 2 deletions Sources/ManageSmileys.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,9 @@ function EditSmileySets()
$set_paths = explode(',', $modSettings['smiley_sets_known']);
$set_names = explode("\n", $modSettings['smiley_sets_names']);

foreach (array('smiley_sets_path', 'smiley_sets_name') as $key)
$_POST[$key] = $smcFunc['normalize']($_POST[$key]);

// Create a new smiley set.
if ($_POST['set'] == -1 && isset($_POST['smiley_sets_path']))
{
Expand Down Expand Up @@ -613,6 +616,9 @@ function AddSmiley()
{
checkSession();

foreach (array('smiley_code', 'smiley_filename', 'smiley_description') as $key)
$_POST[$key] = $smcFunc['normalize']($_POST[$key]);

$_POST['smiley_code'] = htmltrim__recursive($_POST['smiley_code']);
$_POST['smiley_location'] = empty($_POST['smiley_location']) || $_POST['smiley_location'] > 2 || $_POST['smiley_location'] < 0 ? 0 : (int) $_POST['smiley_location'];
$_POST['smiley_filename'] = htmltrim__recursive($_POST['smiley_filename']);
Expand Down Expand Up @@ -1005,6 +1011,9 @@ function EditSmileys()
// Otherwise an edit.
else
{
foreach (array('smiley_code', 'smiley_description') as $key)
$_POST[$key] = $smcFunc['normalize']($_POST[$key]);

$_POST['smiley'] = (int) $_POST['smiley'];
$_POST['smiley_code'] = htmltrim__recursive($_POST['smiley_code']);
$_POST['smiley_location'] = empty($_POST['smiley_location']) || $_POST['smiley_location'] > 2 || $_POST['smiley_location'] < 0 ? 0 : (int) $_POST['smiley_location'];
Expand Down Expand Up @@ -1068,8 +1077,8 @@ function EditSmileys()
$filenames = array();
foreach ($_POST['smiley_filename'] as $posted_set => $posted_filename)
{
$posted_set = htmltrim__recursive($posted_set);
$posted_filename = htmltrim__recursive($posted_filename);
$posted_set = $smcFunc['htmltrim']($smcFunc['normalize']($posted_set));
$posted_filename = $smcFunc['htmltrim']($smcFunc['normalize']($posted_filename));

// Make sure the set already exists.
if (!in_array($posted_set, $known_sets))
Expand Down Expand Up @@ -2224,6 +2233,9 @@ function EditMessageIcons()
{
$_GET['icon'] = (int) $_GET['icon'];

foreach (array('icon_filename', 'icon_description') as $key)
$_POST[$key] = $smcFunc['normalize']($_POST[$key]);

// Do some preperation with the data... like check the icon exists *somewhere*
if (strpos($_POST['icon_filename'], '.png') !== false)
$_POST['icon_filename'] = substr($_POST['icon_filename'], 0, -4);
Expand Down
2 changes: 1 addition & 1 deletion Sources/Post.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1966,7 +1966,7 @@ function Post2()
// If the poster is a guest evaluate the legality of name and email.
if ($posterIsGuest)
{
$_POST['guestname'] = !isset($_POST['guestname']) ? '' : trim($_POST['guestname']);
$_POST['guestname'] = !isset($_POST['guestname']) ? '' : trim(normalize_spaces(sanitize_chars($_POST['guestname'], 1, ' '), true, true, array('no_breaks' => true, 'replace_tabs' => true, 'collapse_hspace' => true)));
$_POST['email'] = !isset($_POST['email']) ? '' : trim($_POST['email']);

if ($_POST['guestname'] == '' || $_POST['guestname'] == '_')
Expand Down
12 changes: 7 additions & 5 deletions Sources/Profile-Modify.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ function $preload: A function that is used to load data required for this eleme
resetPassword($context['id_member'], $value);
elseif ($value !== null)
{
validateUsername($context['id_member'], trim(preg_replace('~[\t\n\r \x0B\0' . ($context['utf8'] ? '\x{A0}\x{AD}\x{2000}-\x{200F}\x{201F}\x{202F}\x{3000}\x{FEFF}' : '\x00-\x08\x0B\x0C\x0E-\x19\xA0') . ']+~' . ($context['utf8'] ? 'u' : ''), ' ', $value)));
validateUsername($context['id_member'], trim(normalize_spaces(sanitize_chars($value, 1, ' '), true, true, array('no_breaks' => true, 'replace_tabs' => true, 'collapse_hspace' => true))));
updateMemberData($context['id_member'], array('member_name' => $value));

// Call this here so any integrated systems will know about the name change (resetPassword() takes care of this if we're letting SMF generate the password)
Expand Down Expand Up @@ -409,7 +409,7 @@ function $preload: A function that is used to load data required for this eleme
'enabled' => allowedTo('profile_displayed_name_own') || allowedTo('profile_displayed_name_any') || allowedTo('moderate_forum'),
'input_validate' => function(&$value) use ($context, $smcFunc, $sourcedir, $cur_profile)
{
$value = trim(preg_replace('~[\t\n\r \x0B\0' . ($context['utf8'] ? '\x{A0}\x{AD}\x{2000}-\x{200F}\x{201F}\x{202F}\x{3000}\x{FEFF}' : '\x00-\x08\x0B\x0C\x0E-\x19\xA0') . ']+~' . ($context['utf8'] ? 'u' : ''), ' ', $value));
$value = trim(normalize_spaces(sanitize_chars($value, 1, ' '), true, true, array('no_breaks' => true, 'replace_tabs' => true, 'collapse_hspace' => true)));

if (trim($value) == '')
return 'no_name';
Expand Down Expand Up @@ -795,7 +795,7 @@ function setupProfileContext($fields)
*/
function saveProfileFields()
{
global $profile_fields, $profile_vars, $context, $old_profile, $post_errors, $cur_profile;
global $profile_fields, $profile_vars, $context, $old_profile, $post_errors, $cur_profile, $smcFunc;

// Load them up.
loadProfileFields();
Expand All @@ -817,6 +817,8 @@ function saveProfileFields()
if (!isset($_POST[$key]) || !empty($field['is_dummy']) || (isset($_POST['preview_signature']) && $key == 'signature'))
continue;

$_POST[$key] = sanitize_chars($smcFunc['normalize']($_POST[$key]), in_array($key, array('member_name', 'real_name')) ? 1 : 0);

// What gets updated?
$db_key = isset($field['save_key']) ? $field['save_key'] : $key;

Expand Down Expand Up @@ -3143,7 +3145,7 @@ function profileLoadGroups()
*/
function profileLoadSignatureData()
{
global $modSettings, $context, $txt, $cur_profile, $memberContext;
global $modSettings, $context, $txt, $cur_profile, $memberContext, $smcFunc;

// Signature limits.
list ($sig_limits, $sig_bbc) = explode(':', $modSettings['signature_settings']);
Expand Down Expand Up @@ -3173,7 +3175,7 @@ function profileLoadSignatureData()
$context['member']['signature'] = empty($cur_profile['signature']) ? '' : str_replace(array('<br>', '<', '>', '"', '\''), array("\n", '&lt;', '&gt;', '&quot;', '&#039;'), $cur_profile['signature']);
else
{
$signature = !empty($_POST['signature']) ? $_POST['signature'] : '';
$signature = $_POST['signature'] = !empty($_POST['signature']) ? $smcFunc['normalize']($_POST['signature']) : '';
$validation = profileValidateSignature($signature);
if (empty($context['post_errors']))
{
Expand Down
9 changes: 6 additions & 3 deletions Sources/Register.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,8 +317,11 @@ function Register2()
$_POST,
function (&$value, $key) use ($context, $smcFunc)
{
// Replace any kind of space with a normal space, and remove any kind of control character, then trim.
$value = $smcFunc['htmltrim'](preg_replace(array('~[\h\v]+~' . ($context['utf8'] ? 'u' : ''), '~\p{Cc}+~'), array(' ', ''), $value));
// Normalize Unicode characters. (Does nothing if not in UTF-8 mode.)
$value = $smcFunc['normalize']($value);

// Replace any kind of space or illegal character with a normal space, and then trim.
$value = $smcFunc['htmltrim'](normalize_spaces(sanitize_chars($value, 1, ' '), true, true, array('no_breaks' => true, 'replace_tabs' => true, 'collapse_hspace' => true)));
}
);

Expand Down Expand Up @@ -905,7 +908,7 @@ function RegisterCheckUsername()
$context['valid_username'] = true;

// Clean it up like mother would.
$context['checked_username'] = preg_replace('~[\t\n\r \x0B\0' . ($context['utf8'] ? '\x{A0}\x{AD}\x{2000}-\x{200F}\x{201F}\x{202F}\x{3000}\x{FEFF}' : '\x00-\x08\x0B\x0C\x0E-\x19\xA0') . ']+~' . ($context['utf8'] ? 'u' : ''), ' ', $context['checked_username']);
$context['checked_username'] = trim(normalize_spaces(sanitize_chars($context['checked_username'], 1, ' '), true, true, array('no_breaks' => true, 'replace_tabs' => true, 'collapse_hspace' => true)));

require_once($sourcedir . '/Subs-Auth.php');
$errors = validateUsername(0, $context['checked_username'], true);
Expand Down
Loading