fix: under windows, librime doesn't support resource_id(file name) with non-ANSI characters, which is supported by code page.

[fxliang]

Pull request

Issue tracker

Fixes will automatically close the related issue

Fixes #

under windows, librime doesn't support resource_id(file name) with non-ANSI characters, which is supported by code page.

Feature

Describe feature of pull request

Unit test

• Done

Manual test

• Done

Code Review

1. Unit and manual test pass
2. GitHub Action CI pass
3. At least one contributor reviews and votes
4. Can be merged clean without conflicts
5. PR will be merged by rebase upstream base

Additional Info

[fxliang]

@lotem
under windows with msvc, path initialized by std::string will get wrong result if there are characters not supported by the current code page of Windows system, but get right result by initializing the path with std::wstring.

here is a sample test

#include <iostream>
#include <string>
#include <filesystem>
#include <windows.h>

static std::wstring U8ToU16(const std::string& str) {
	std::wstring ret;
	int length = static_cast<int>(str.length());
	int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
	if (convcnt > 0) {
		ret.resize(convcnt);
		MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
	}
	return ret;
}

int main() {
	// with a file name ς.txt in the same directory
	// whereas "ς" is not a character supported by cp936
	std::string xlfn = u8"ς";
	std::cout << "file name no ext: " << xlfn << std::endl;
	std::cout << "-----------------------------------------" << std::endl;
	// path with wstirng
	std::wstring xlfnw = U8ToU16(xlfn);
	std::filesystem::path xlpathw(U8ToU16(".\\") + xlfnw + U8ToU16(".txt"));
	bool chk = std::filesystem::exists(xlpathw);
	std::cout << "path by wstring, file exists ? " << (chk ? "yes" : "no") << std::endl;
	std::cout << "-----------------------------------------" << std::endl;
	// path with stirng
	std::filesystem::path xlpath(".\\" + xlfn + ".txt");
	chk = std::filesystem::exists(xlpath);
	std::cout << "path by string, file exists ? " << (chk ? "yes" : "no") << std::endl;

	return 0;
}

If acceptable, the final solution would be( to be tested )

update: failed and don't know why. : (
update: failed caused by ifstream/istream/stringstream/yaml-cpp and so on.

#ifdef _MSC_VER
static std::wstring U8ToU16(const std::string& str) {
  std::wstring ret;
  int length = static_cast<int>(str.length());
  int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
  if (convcnt > 0) {
    ret.resize(convcnt);
    MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
  }
  return ret;
}
#endif /* _MSC_VER*/

std::filesystem::path ResourceResolver::ResolvePath(const string& resource_id) {
#ifdef _MSC_VER
  // use std::wstring, avoiding codepage issue
  return std::filesystem::absolute(
      root_path_ /
      std::filesystem::path(U8ToU16(type_.prefix) + U8ToU16(resource_id) +
                            U8ToU16(type_.suffix)));
#else
  return std::filesystem::absolute(
      root_path_ /
      std::filesystem::path(type_.prefix + resource_id + type_.suffix));
#endif
}

std::filesystem::path FallbackResourceResolver::ResolvePath(
    const string& resource_id) {
  auto default_path = ResourceResolver::ResolvePath(resource_id);
  if (!std::filesystem::exists(default_path)) {
#ifdef _MSC_VER
    // use std::wstring, avoiding codepage issue
    auto fallback_path = std::filesystem::absolute(
        fallback_root_path_ /
        std::filesystem::path(U8ToU16(type_.prefix) + U8ToU16(resource_id) +
                              U8ToU16(type_.suffix)));
#else
    auto fallback_path = std::filesystem::absolute(
        fallback_root_path_ /
        std::filesystem::path(type_.prefix + resource_id + type_.suffix));
#endif
    if (std::filesystem::exists(fallback_path)) {
      return fallback_path;
    }
  }
  return default_path;
}