first commit

This commit is contained in:
mandreyel 2017-09-04 18:24:40 +02:00
commit 4583a6529e
6 changed files with 792 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
test/**
!test/test.cpp

View File

@ -0,0 +1,146 @@
#ifndef MIO_BASIC_MMAP_HEADER
#define MIO_BASIC_MMAP_HEADER
#ifdef _WIN32
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif // WIN32_LEAN_AND_MEAN
# include <windows.h>
#else // ifdef _WIN32
# define INVALID_HANDLE_VALUE -1
#endif // ifdef _WIN32
#include <iterator>
#include <string>
namespace mio {
namespace detail {
size_t page_size();
/**
* Most of the logic in establishing a memory mapping is the same for both read-only and
* read-write mappings, so they both inherit from basic_mmap.
*/
template<typename CharT> struct basic_mmap
{
using value_type = CharT;
using size_type = int64_t;
using reference = value_type&;
using const_reference = const reference;
using pointer = value_type*;
using const_pointer = const pointer;
using difference_type = std::ptrdiff_t;
using iterator = pointer;
using const_iterator = const_pointer;
// TODO these don't seem to work with regular pointers
//using reverse_iterator = std::reverse_iterator<iterator>;
//using const_reverse_iterator = std::reverse_iterator<const_iterator>;
using iterator_category = std::random_access_iterator_tag;
#ifdef _WIN32
using handle_type = HANDLE;
#else
using handle_type = int;
#endif
enum class access_mode
{
read_only,
read_write
};
private:
// Points to the first requested byte, and not to the actual start of the mapping.
pointer data_ = nullptr;
// On POSIX, we only need a file handle to create a mapping, while on Windows
// systems the file handle is necessary to retrieve a file mapping handle, but any
// subsequent operations on the mapped region must be done through the file mapping
// handle.
handle_type file_handle_ = INVALID_HANDLE_VALUE;
#if defined(_WIN32)
handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE;
#endif
// Length requested by user, which may not be the length of the full mapping.
size_type length_ = 0;
size_type mapped_length_ = 0;
public:
basic_mmap() = default;
basic_mmap(const basic_mmap&) = delete;
basic_mmap& operator=(const basic_mmap&) = delete;
basic_mmap(basic_mmap&&);
basic_mmap& operator=(basic_mmap&&);
~basic_mmap();
/**
* On *nix systems is_open and is_mapped are the same and don't actually say if
* the file itself is open or closed, they only refer to the mapping. This is
* because a mapping remains valid (as long as it's not unmapped) even if another
* entity closes the file which is being mapped.
*
* On Windows, however, in order to map a file, both an active file handle and a
* mapping handle is required, so is_open checks for a valid file handle, while
* is_mapped checks for a valid mapping handle.
*/
bool is_open() const noexcept;
bool is_mapped() const noexcept;
bool empty() const noexcept { return length() == 0; }
/**
* size/length returns the logical length (i.e. the one user requested), while
* mapped_length returns the actual mapped length, which is usually a multiple of
* the OS' page size.
*/
size_type size() const noexcept { return length_; }
size_type length() const noexcept { return length_; }
size_type mapped_length() const noexcept { return mapped_length_; }
pointer data() noexcept { return data_; }
const_pointer data() const noexcept { return data_; }
iterator begin() noexcept { return data(); }
const_iterator begin() const noexcept { return data(); }
const_iterator cbegin() const noexcept { return data(); }
iterator end() noexcept { return begin() + length(); }
const_iterator end() const noexcept { return begin() + length(); }
const_iterator cend() const noexcept { return begin() + length(); }
reference operator[](const size_type i) noexcept { return data_[i]; }
const_reference operator[](const size_type i) const noexcept { return data_[i]; }
void map(const std::string& path, const size_type offset, const size_type length,
const access_mode mode, std::error_code& error);
void map(const handle_type handle, const size_type offset, const size_type length,
const access_mode mode, std::error_code& error);
void unmap();
void sync(std::error_code& error);
private:
static handle_type open_file(const std::string& path,
const access_mode mode, std::error_code& error);
pointer get_mapping_start() noexcept;
/** NOTE: file_handle_ must be valid. */
size_type query_file_size(std::error_code& error) noexcept;
void map(const size_type offset, const size_type length,
const access_mode mode, std::error_code& error);
void verify_file_handle(std::error_code& error) const noexcept;
};
} // namespace detail
} // namespace mio
#include "basic_mmap.ipp"
#endif // MIO_BASIC_MMAP_HEADER

View File

@ -0,0 +1,336 @@
#ifndef MIO_BASIC_MMAP_IMPL
#define MIO_BASIC_MMAP_IMPL
#include "basic_mmap.hpp"
#ifndef _WIN32
# include <unistd.h>
# include <fcntl.h>
# include <sys/mman.h>
# include <sys/stat.h>
#endif
namespace mio {
namespace detail {
#if defined(_WIN32)
inline DWORD int64_high(int64_t n) noexcept
{
return n >> 32;
}
inline DWORD int64_low(int64_t n) noexcept
{
return n & 0xFF'FF'FF'FF;
}
#endif
size_t page_size()
{
static const size_t page_size = []
{
#ifdef _WIN32
SYSTEM_INFO SystemInfo;
GetSystemInfo(&SystemInfo);
return SystemInfo.dwAllocationGranularity;
#else
return sysconf(_SC_PAGE_SIZE);
#endif
}();
return page_size;
}
inline size_t make_page_aligned(size_t offset) noexcept
{
const static size_t page_size_ = page_size();
// use integer division to round down to the nearest page alignment
return offset / page_size_ * page_size_;
}
inline std::error_code last_error() noexcept
{
std::error_code error;
#ifdef _WIN32
error.assign(GetLastError(), std::system_category());
#else
error.assign(errno, std::system_category());
#endif
return error;
}
// -- basic_mmap --
template<typename CharT>
basic_mmap<CharT>::~basic_mmap()
{
unmap();
}
template<typename CharT>
basic_mmap<CharT>::basic_mmap(basic_mmap&& other)
: data_(std::move(other.data_))
, length_(std::move(other.length_))
, mapped_length_(std::move(other.mapped_length_))
, file_handle_(std::move(other.file_handle_))
#ifdef _WIN32
, file_mapping_handle_(std::move(other.file_mapping_handle_))
#endif
{
other.data_ = nullptr;
other.length_ = other.mapped_length_ = 0;
other.file_handle_ = INVALID_HANDLE_VALUE;
#ifdef _WIN32
other.file_mapping_handle_ = INVALID_HANDLE_VALUE;
#endif
}
template<typename CharT>
basic_mmap<CharT>& basic_mmap<CharT>::operator=(basic_mmap&& other)
{
if(this != &other)
{
data_ = std::move(other.data_);
length_ = std::move(other.length_);
mapped_length_ = std::move(other.mapped_length_);
file_handle_ = std::move(other.file_handle_);
#ifdef _WIN32
file_mapping_handle_ = std::move(other.file_mapping_handle_);
#endif
other.data_ = nullptr;
other.length_ = other.mapped_length_ = 0;
other.file_handle_ = INVALID_HANDLE_VALUE;
#ifdef _WIN32
other.file_mapping_handle_ = INVALID_HANDLE_VALUE;
#endif
}
return *this;
}
template<typename CharT>
void basic_mmap<CharT>::map(const std::string& path, const size_type offset,
const size_type length, const access_mode mode, std::error_code& error)
{
if(!path.empty() && !is_open())
{
error.clear();
const auto handle = open_file(path, mode, error);
if(error) { return; }
map(handle, offset, length, mode, error);
}
}
template<typename CharT>
typename basic_mmap<CharT>::handle_type
basic_mmap<CharT>::open_file(const std::string& path,
const basic_mmap<CharT>::access_mode mode, std::error_code& error)
{
#if defined(_WIN32)
const auto handle = ::CreateFile(path.c_str(),
mode == basic_mmap<CharT>::access_mode::read_only
? GENERIC_READ : GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
0,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
0);
if(handle == INVALID_HANDLE_VALUE)
{
error = last_error();
}
#else
const auto handle = ::open(path.c_str(),
mode == basic_mmap<CharT>::access_mode::read_only ? O_RDONLY : O_RDWR);
if(handle == -1)
{
error = last_error();
}
#endif
return handle;
}
template<typename CharT>
void basic_mmap<CharT>::map(const handle_type handle, const size_type offset,
const size_type length, const access_mode mode, std::error_code& error)
{
error.clear();
if(handle == INVALID_HANDLE_VALUE)
{
error = std::make_error_code(std::errc::bad_file_descriptor);
return;
}
file_handle_ = handle;
const auto file_size = query_file_size(error);
if(error) { return; }
if(offset + length > file_size)
{
error = std::make_error_code(std::errc::invalid_argument);
return;
}
map(offset, length, mode, error);
}
template<typename CharT>
void basic_mmap<CharT>::map(const size_type offset, const size_type length,
const access_mode mode, std::error_code& error)
{
const size_type aligned_offset = make_page_aligned(offset);
const size_type length_to_map = offset - aligned_offset + length;
#if defined(_WIN32)
const size_type max_file_size = offset + length;
file_mapping_handle_ = ::CreateFileMapping(
file_handle_,
0,
mode == access_mode::read_only ? PAGE_READONLY : PAGE_READWRITE,
int64_high(max_file_size),
int64_low(max_file_size),
0);
if(file_mapping_handle_ == INVALID_HANDLE_VALUE)
{
error = last_error();
return;
}
const pointer mapping_start = static_cast<pointer>(::MapViewOfFile(
file_mapping_handle_,
mode == access_mode::read_only ? FILE_MAP_READ : FILE_MAP_WRITE,
int64_high(aligned_offset),
int64_low(aligned_offset),
length_to_map));
if(mapping_start == nullptr)
{
error = last_error();
return;
}
#else
const pointer mapping_start = static_cast<pointer>(::mmap(
0, // don't give hint as to where to map
length_to_map,
mode == basic_mmap<CharT>::access_mode::read_only ? PROT_READ : PROT_WRITE,
MAP_SHARED, // TODO do we want to share it?
file_handle_,
aligned_offset));
if(mapping_start == MAP_FAILED)
{
error = last_error();
return;
}
#endif
data_ = mapping_start + offset - aligned_offset;
length_ = length;
mapped_length_ = length_to_map;
}
template<typename CharT>
void basic_mmap<CharT>::sync(std::error_code& error)
{
error.clear();
verify_file_handle(error);
if(error) { return; }
if(data() != nullptr)
{
#ifdef _WIN32
if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0
|| ::FlushFileBuffers(file_handle_) == 0)
#else
if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0)
#endif
{
error = last_error();
return;
}
}
#ifdef _WIN32
if(::FlushFileBuffers(file_handle_) == 0)
{
error = last_error();
}
#endif
}
template<typename CharT>
void basic_mmap<CharT>::unmap()
{
// TODO do we care about errors here?
if((data_ != nullptr) && (file_handle_ != INVALID_HANDLE_VALUE))
{
#ifdef _WIN32
::UnmapViewOfFile(get_mapping_start());
::CloseHandle(file_mapping_handle_);
file_mapping_handle_ = INVALID_HANDLE_VALUE;
#else
::munmap(const_cast<pointer>(get_mapping_start()), mapped_length_);
#endif
}
data_ = nullptr;
length_ = mapped_length_ = 0;
file_handle_ = INVALID_HANDLE_VALUE;
#ifdef _WIN32
file_mapping_handle_ = INVALID_HANDLE_VALUE;
#endif
}
template<typename CharT>
typename basic_mmap<CharT>::size_type
basic_mmap<CharT>::query_file_size(std::error_code& error) noexcept
{
#ifdef _WIN32
PLARGE_INTEGER file_size;
if(::GetFileSizeEx(file_handle_, &file_size) == 0)
{
error = last_error();
return 0;
}
return file_size;
#else
struct stat sbuf;
if(::fstat(file_handle_, &sbuf) == -1)
{
error = last_error();
return 0;
}
return sbuf.st_size;
#endif
}
template<typename CharT>
typename basic_mmap<CharT>::pointer basic_mmap<CharT>::get_mapping_start() noexcept
{
if(!data_) { return nullptr; }
const auto offset = mapped_length_ - length_;
return data_ - offset;
}
template<typename CharT>
void basic_mmap<CharT>::verify_file_handle(std::error_code& error) const noexcept
{
if(!is_open() || !is_mapped())
{
error = std::make_error_code(std::errc::bad_file_descriptor);
}
}
template<typename CharT>
bool basic_mmap<CharT>::is_open() const noexcept
{
return file_handle_ != INVALID_HANDLE_VALUE;
}
template<typename CharT>
bool basic_mmap<CharT>::is_mapped() const noexcept
{
#ifdef _WIN32
return file_mapping_handle_ != INVALID_HANDLE_VALUE;
#else
return is_open();
#endif
}
} // namespace detail
} // namespace mio
#endif // MIO_BASIC_MMAP_IMPL

View File

@ -0,0 +1,81 @@
#ifndef MIO_MMAP_IMPL
#define MIO_MMAP_IMPL
#include "../mmap.hpp"
namespace mio {
// -- basic_mmap_source --
template<typename CharT>
basic_mmap_source<CharT>::basic_mmap_source(const std::string& path,
const size_type offset, const size_type length)
{
std::error_code error;
map(path, offset, length, error);
if(error) { throw error; }
}
template<typename CharT>
basic_mmap_source<CharT>::basic_mmap_source(const handle_type handle,
const size_type offset, const size_type length)
{
std::error_code error;
map(handle, offset, length, error);
if(error) { throw error; }
}
template<typename CharT>
void basic_mmap_source<CharT>::map(const std::string& path, const size_type offset,
const size_type length, std::error_code& error)
{
impl_.map(path, offset, length, impl_type::access_mode::read_only, error);
}
template<typename CharT>
void basic_mmap_source<CharT>::map(const handle_type handle, const size_type offset,
const size_type length, std::error_code& error)
{
impl_.map(handle, offset, length, impl_type::access_mode::read_only, error);
}
// -- basic_mmap_sink --
template<typename CharT>
basic_mmap_sink<CharT>::basic_mmap_sink(const std::string& path,
const size_type offset, const size_type length)
{
std::error_code error;
map(path, offset, length, error);
if(error) { throw error; }
}
template<typename CharT>
basic_mmap_sink<CharT>::basic_mmap_sink(const handle_type handle,
const size_type offset, const size_type length)
{
std::error_code error;
map(handle, offset, length, error);
if(error) { throw error; }
}
template<typename CharT>
void basic_mmap_sink<CharT>::map(const std::string& path, const size_type offset,
const size_type length, std::error_code& error)
{
impl_.map(path, offset, length, impl_type::access_mode::read_only, error);
}
template<typename CharT>
void basic_mmap_sink<CharT>::map(const handle_type handle, const size_type offset,
const size_type length, std::error_code& error)
{
impl_.map(handle, offset, length, impl_type::access_mode::read_write, error);
}
template<typename CharT>
void basic_mmap_sink<CharT>::sync(std::error_code& error) { impl_.sync(error); }
} // namespace mio
#endif // MIO_MMAP_IMPL

185
include/mio/mmap.hpp Normal file
View File

@ -0,0 +1,185 @@
#ifndef MIO_MMAP_HEADER
#define MIO_MMAP_HEADER
#include "detail/basic_mmap.hpp"
namespace mio {
/**
* When specifying a file to map, there is no need to worry about providing
* offsets that are aligned with the operating system's page granularity, this is taken
* care of within the class in both cases.
*
* Both classes have std::unique_ptr<> semantics, thus only a single entity may own
* a mapping to a file at any given time.
*
* Remapping a file is possible, but unmap must be called before that.
*
* For now, both classes may only be used with an existing open file by providing the
* file's handle.
*
* Both classes' destructors unmap the file. However, mmap_sink's destructor does not
* sync the mapped file view to disk, this has to be done manually with a call tosink.
*/
/** A read-only file memory mapping. */
template<typename CharT> struct basic_mmap_source
{
using impl_type = detail::basic_mmap<CharT>;
using value_type = typename impl_type::value_type;
using size_type = typename impl_type::size_type;
using reference = typename impl_type::reference;
using const_reference = typename impl_type::const_reference;
using pointer = typename impl_type::pointer;
using const_pointer = typename impl_type::const_pointer;
using difference_type = typename impl_type::difference_type;
using iterator = typename impl_type::iterator;
using const_iterator = typename impl_type::const_iterator;
//using reverse_iterator = std::reverse_iterator<iterator>;
//using const_reverse_iterator = std::reverse_iterator<const_iterator>;
using iterator_category = typename impl_type::iterator_category;
using handle_type = typename impl_type::handle_type;
private:
impl_type impl_;
public:
basic_mmap_source() = default;
basic_mmap_source(const std::string& path,
const size_type offset, const size_type length);
basic_mmap_source(const handle_type handle,
const size_type offset, const size_type length);
/**
* On *nix systems is_open and is_mapped are the same and don't actually say if
* the file itself is open or closed, they only refer to the mapping. This is
* because a mapping remains valid (as long as it's not unmapped) even if another
* entity closes the file which is being mapped.
*
* On Windows, however, in order to map a file, both an active file handle and a
* mapping handle is required, so is_open checks for a valid file handle, while
* is_mapped checks for a valid mapping handle.
*/
bool is_open() const noexcept { return impl_.is_open(); }
bool is_mapped() const noexcept { return impl_.is_mapped(); }
bool empty() const noexcept { return impl_.is_empty(); }
/**
* size/length returns the logical length (i.e. the one user requested), while
* mapped_length returns the actual mapped length, which is usually a multiple of
* the OS' page size.
*/
size_type size() const noexcept { return impl_.size(); }
size_type length() const noexcept { return impl_.length(); }
size_type mapped_length() const noexcept { return impl_.mapped_length(); }
const_pointer data() const noexcept { return impl_.data(); }
const_iterator begin() const noexcept { return impl_.begin(); }
const_iterator cbegin() const noexcept { return impl_.cbegin(); }
const_iterator end() const noexcept { return impl_.end(); }
const_iterator cend() const noexcept { return impl_.cend(); }
const_reference operator[](const size_type i) const noexcept { return impl_[i]; }
void map(const std::string& path, const size_type offset,
const size_type length, std::error_code& error);
void map(const handle_type handle, const size_type offset,
const size_type length, std::error_code& error);
void unmap() { impl_.unmap(); }
};
/** A read-write file memory mapping. */
template<typename CharT> struct basic_mmap_sink
{
using impl_type = detail::basic_mmap<CharT>;
using value_type = typename impl_type::value_type;
using size_type = typename impl_type::size_type;
using reference = typename impl_type::reference;
using const_reference = typename impl_type::const_reference;
using pointer = typename impl_type::pointer;
using const_pointer = typename impl_type::const_pointer;
using difference_type = typename impl_type::difference_type;
using iterator = typename impl_type::iterator;
using const_iterator = typename impl_type::const_iterator;
//using reverse_iterator = std::reverse_iterator<iterator>;
//using const_reverse_iterator = std::reverse_iterator<const_iterator>;
using iterator_category = typename impl_type::iterator_category;
using handle_type = typename impl_type::handle_type;
private:
impl_type impl_;
public:
basic_mmap_sink() = default;
basic_mmap_sink(const std::string& path,
const size_type offset, const size_type length);
basic_mmap_sink(const handle_type handle,
const size_type offset, const size_type length);
/**
* On *nix systems is_open and is_mapped are the same and don't actually say if
* the file itself is open or closed, they only refer to the mapping. This is
* because a mapping remains valid (as long as it's not unmapped) even if another
* entity closes the file which is being mapped.
*
* On Windows, however, in order to map a file, both an active file handle and a
* mapping handle is required, so is_open checks for a valid file handle, while
* is_mapped checks for a valid mapping handle.
*/
bool is_open() const noexcept { return impl_.is_open(); }
bool is_mapped() const noexcept { return impl_.is_mapped(); }
bool empty() const noexcept { return impl_.is_empty(); }
/**
* size/length returns the logical length (i.e. the one user requested), while
* mapped_length returns the actual mapped length, which is usually a multiple of
* the OS' page size.
*/
size_type size() const noexcept { return impl_.size(); }
size_type length() const noexcept { return impl_.length(); }
size_type mapped_length() const noexcept { return impl_.mapped_length(); }
pointer data() noexcept { return impl_.data(); }
const_pointer data() const noexcept { return impl_.data(); }
iterator begin() noexcept;
const_iterator begin() const noexcept { return impl_.begin(); }
const_iterator cbegin() const noexcept { return impl_.cbegin(); }
iterator end() noexcept;
const_iterator end() const noexcept { return impl_.end(); }
const_iterator cend() const noexcept { return impl_.cend(); }
reference operator[](const size_type i) noexcept { return impl_[i]; }
const_reference operator[](const size_type i) const noexcept { return impl_[i]; }
void map(const std::string& path, const size_type offset,
const size_type length, std::error_code& error);
void map(const handle_type handle, const size_type offset,
const size_type length, std::error_code& error);
/** Flushes the memory mapped page to disk. */
void sync(std::error_code& error);
void unmap() { impl_.unmap(); }
};
using mmap_sink = basic_mmap_sink<char>;
using mmap_source = basic_mmap_source<char>;
using wmmap_sink = basic_mmap_sink<wchar_t>;
using wmmap_source = basic_mmap_source<wchar_t>;
using u16mmap_sink = basic_mmap_sink<char16_t>;
using u16mmap_source = basic_mmap_source<char16_t>;
using u32mmap_sink = basic_mmap_sink<char32_t>;
using u32mmap_source = basic_mmap_source<char32_t>;
} // namespace mio
#include "detail/mmap.ipp"
#endif // MIO_MMAP_HEADER

42
test/test.cpp Normal file
View File

@ -0,0 +1,42 @@
#include "../include/mio/mmap.hpp"
#include <string>
#include <fstream>
#include <cstdlib>
#include <cassert>
#include <system_error>
int main(int argc, char** argv)
{
const char* path = argc >= 2 ? argv[1] : ".";
std::error_code error;
std::string buffer(0x4000 - 250, 'M');
std::ofstream file(path);
file << buffer;
file.close();
mio::mmap_source file_view;
file_view.map(path, 0, buffer.size(), error);
if(error)
{
const auto& errmsg = error.message();
std::printf("error: %s, exiting...\n", errmsg.c_str());
return error.value();
}
assert(file_view.is_open());
assert(file_view.is_mapped());
assert(file_view.size() == buffer.size());
for(auto i = 0; i < buffer.size(); ++i)
{
if(file_view[i] != buffer[i])
{
std::printf("%ith byte mismatch: expected(%i) <> actual(%i)",
i, buffer[i], file_view[i]);
assert(0);
}
}
}