Skip to content

Adds unicode support to Windows environment. #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Adds unicode support to Windows environment. #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3b8f3da
Adds unicode support to Windows environment.
sipsorcery Dec 12, 2019
594049f
Merge branch 'master' into windows_unicode
sipsorcery Sep 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 70 additions & 28 deletions util/env_windows.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,9 @@ class WindowsEnv : public Env {
*result = nullptr;
DWORD desired_access = GENERIC_READ;
DWORD share_mode = FILE_SHARE_READ;
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -392,8 +393,9 @@ class WindowsEnv : public Env {
*result = nullptr;
DWORD desired_access = GENERIC_READ;
DWORD share_mode = FILE_SHARE_READ;
auto wFilename = toUtf16(filename);
ScopedHandle handle =
::CreateFileA(filename.c_str(), desired_access, share_mode,
::CreateFileW(wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_EXISTING,
FILE_ATTRIBUTE_READONLY,
/*hTemplateFile=*/nullptr);
Expand All @@ -413,11 +415,12 @@ class WindowsEnv : public Env {
}

ScopedHandle mapping =
::CreateFileMappingA(handle.get(),
/*security attributes=*/nullptr, PAGE_READONLY,
/*dwMaximumSizeHigh=*/0,
/*dwMaximumSizeLow=*/0,
/*lpName=*/nullptr);
::CreateFileMappingW(handle.get(),
/*security attributes=*/nullptr,
PAGE_READONLY,
/*dwMaximumSizeHigh=*/0,
/*dwMaximumSizeLow=*/0,
/*lpName=*/nullptr);
if (mapping.is_valid()) {
void* mmap_base = ::MapViewOfFile(mapping.get(), FILE_MAP_READ,
/*dwFileOffsetHigh=*/0,
Expand All @@ -438,8 +441,9 @@ class WindowsEnv : public Env {
WritableFile** result) override {
DWORD desired_access = GENERIC_WRITE;
DWORD share_mode = 0; // Exclusive access.
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -455,8 +459,9 @@ class WindowsEnv : public Env {
WritableFile** result) override {
DWORD desired_access = FILE_APPEND_DATA;
DWORD share_mode = 0; // Exclusive access.
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -469,14 +474,16 @@ class WindowsEnv : public Env {
}

bool FileExists(const std::string& filename) override {
return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES;
auto wFilename = toUtf16(filename);
return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES;
}

Status GetChildren(const std::string& directory_path,
std::vector<std::string>* result) override {
const std::string find_pattern = directory_path + "\\*";
WIN32_FIND_DATAA find_data;
HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data);
WIN32_FIND_DATAW find_data;
auto wFind_pattern = toUtf16(find_pattern);
HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data);
if (dir_handle == INVALID_HANDLE_VALUE) {
DWORD last_error = ::GetLastError();
if (last_error == ERROR_FILE_NOT_FOUND) {
Expand All @@ -488,11 +495,12 @@ class WindowsEnv : public Env {
char base_name[_MAX_FNAME];
char ext[_MAX_EXT];

if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name,
auto find_data_filename = toUtf8(find_data.cFileName);
if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, base_name,
ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) {
result->emplace_back(std::string(base_name) + ext);
}
} while (::FindNextFileA(dir_handle, &find_data));
} while (::FindNextFileW(dir_handle, &find_data));
DWORD last_error = ::GetLastError();
::FindClose(dir_handle);
if (last_error != ERROR_NO_MORE_FILES) {
Expand All @@ -502,29 +510,33 @@ class WindowsEnv : public Env {
}

Status RemoveFile(const std::string& filename) override {
if (!::DeleteFileA(filename.c_str())) {
auto wFilename = toUtf16(filename);
if (!::DeleteFileW(wFilename.c_str())) {
return WindowsError(filename, ::GetLastError());
}
return Status::OK();
}

Status CreateDir(const std::string& dirname) override {
if (!::CreateDirectoryA(dirname.c_str(), nullptr)) {
auto wDirname = toUtf16(dirname);
if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) {
return WindowsError(dirname, ::GetLastError());
}
return Status::OK();
}

Status RemoveDir(const std::string& dirname) override {
if (!::RemoveDirectoryA(dirname.c_str())) {
auto wDirname = toUtf16(dirname);
if (!::RemoveDirectoryW(wDirname.c_str())) {
return WindowsError(dirname, ::GetLastError());
}
return Status::OK();
}

Status GetFileSize(const std::string& filename, uint64_t* size) override {
WIN32_FILE_ATTRIBUTE_DATA file_attributes;
if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard,
auto wFilename = toUtf16(filename);
if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard,
&file_attributes)) {
return WindowsError(filename, ::GetLastError());
}
Expand All @@ -538,7 +550,9 @@ class WindowsEnv : public Env {
Status RenameFile(const std::string& from, const std::string& to) override {
// Try a simple move first. It will only succeed when |to| doesn't already
// exist.
if (::MoveFileA(from.c_str(), to.c_str())) {
auto wFrom = toUtf16(from);
auto wTo = toUtf16(to);
if (::MoveFileW(wFrom.c_str(), wTo.c_str())) {
return Status::OK();
}
DWORD move_error = ::GetLastError();
Expand All @@ -547,7 +561,7 @@ class WindowsEnv : public Env {
// succeed when |to| does exist. When writing to a network share, we may not
// be able to change the ACLs. Ignore ACL errors then
// (REPLACEFILE_IGNORE_MERGE_ERRORS).
if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr,
if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr,
REPLACEFILE_IGNORE_MERGE_ERRORS,
/*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) {
return Status::OK();
Expand All @@ -567,8 +581,9 @@ class WindowsEnv : public Env {
Status LockFile(const std::string& filename, FileLock** lock) override {
*lock = nullptr;
Status result;
ScopedHandle handle = ::CreateFileA(
filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
/*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
nullptr);
if (!handle.is_valid()) {
Expand Down Expand Up @@ -608,10 +623,11 @@ class WindowsEnv : public Env {
return Status::OK();
}

char tmp_path[MAX_PATH];
if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) {
wchar_t wtmp_path[MAX_PATH];
if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) {
return WindowsError("GetTempPath", ::GetLastError());
}
std::string tmp_path = toUtf8(std::wstring(wtmp_path));
std::stringstream ss;
ss << tmp_path << "leveldbtest-" << std::this_thread::get_id();
*result = ss.str();
Expand All @@ -622,7 +638,8 @@ class WindowsEnv : public Env {
}

Status NewLogger(const std::string& filename, Logger** result) override {
std::FILE* fp = std::fopen(filename.c_str(), "w");
auto wFilename = toUtf16(filename);
std::FILE* fp = _wfopen(wFilename.c_str(), L"w");
if (fp == nullptr) {
*result = nullptr;
return WindowsError(filename, ::GetLastError());
Expand Down Expand Up @@ -678,6 +695,31 @@ class WindowsEnv : public Env {
GUARDED_BY(background_work_mutex_);

Limiter mmap_limiter_; // Thread-safe.

// Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string.
// See http://utf8everywhere.org/#windows
std::string toUtf8(const std::wstring& wstr) {
if (wstr.empty()) return std::string();
int size_needed = WideCharToMultiByte(
CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
std::string strTo(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0],
size_needed, NULL, NULL);
return strTo;
}

// Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character
// string.
// See http://utf8everywhere.org/#windows
std::wstring toUtf16(const std::string& str) {
if (str.empty()) return std::wstring();
int size_needed = MultiByteToWideChar(
CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0);
std::wstring strTo(size_needed, 0);
MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0],
size_needed);
return strTo;
}
};

// Return the maximum number of concurrent mmaps.
Expand Down
64 changes: 64 additions & 0 deletions util/env_windows_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,70 @@ TEST_F(EnvWindowsTest, TestOpenOnRead) {
ASSERT_LEVELDB_OK(env_->RemoveFile(test_file));
}

TEST_F(EnvWindowsTest, TestOpenOnRead_Unicode) {
// Write some test data to a single file that will be opened |n| times.
std::string test_dir;
ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
std::string test_file = test_dir + u8"/open_on_run🏃_read.txt";

std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::wstring wideUtf8Path = converter.from_bytes(test_file);
FILE* f = _wfopen(wideUtf8Path.c_str(), L"w");
ASSERT_TRUE(f != nullptr);
const char kFileData[] = "abcdefghijklmnopqrstuvwxyz";
fputs(kFileData, f);
fclose(f);

// Open test file some number above the sum of the two limits to force
// leveldb::WindowsEnv to switch from mapping the file into memory
// to basic file reading.
const int kNumFiles = kMMapLimit + 5;
leveldb::RandomAccessFile* files[kNumFiles] = {0};
for (int i = 0; i < kNumFiles; i++) {
ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i]));
}
char scratch;
Slice read_result;
for (int i = 0; i < kNumFiles; i++) {
ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch));
ASSERT_EQ(kFileData[i], read_result[0]);
}
for (int i = 0; i < kNumFiles; i++) {
delete files[i];
}
ASSERT_LEVELDB_OK(env_->DeleteFile(test_file));
}

TEST_F(EnvWindowsTest, TestGetChildrenEmpty) {
// Create some dummy files.
std::string test_dir;
ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));

std::vector<std::string> result;
ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result));
ASSERT_EQ(2, result.size()); // "." and ".." are always returned.
}

TEST_F(EnvWindowsTest, TestGetChildren_ChildFiles) {
// Create some dummy files.
std::string test_dir;
ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));

int childFilesCount = 10;
for (int i = 0; i < childFilesCount; i++) {
std::string test_file = test_dir + u8"/run🏃_and_jump🦘_" + std::to_string(i) + ".txt";
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::wstring wTest_file = converter.from_bytes(test_file);
FILE* f = _wfopen(wTest_file.c_str(), L"w");
ASSERT_TRUE(f != nullptr);
fclose(f);
}

std::vector<std::string> result;
ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result));
ASSERT_EQ(childFilesCount + 2, result.size()); // "." and ".." are returned.
}

} // namespace leveldb

int main(int argc, char** argv) {
Expand Down