+/// std_aligned_alloc() is our wrapper for systems where the c++17 implementation
+/// does not guarantee the availability of aligned_alloc(). Memory allocated with
+/// std_aligned_alloc() must be freed with std_aligned_free().
+
+void* std_aligned_alloc(size_t alignment, size_t size) {
+
+#if defined(POSIXALIGNEDALLOC)
+ void *mem;
+ return posix_memalign(&mem, alignment, size) ? nullptr : mem;
+#elif defined(_WIN32) && !defined(_M_ARM) && !defined(_M_ARM64)
+ return _mm_malloc(size, alignment);
+#elif defined(_WIN32)
+ return _aligned_malloc(size, alignment);
+#else
+ return std::aligned_alloc(alignment, size);
+#endif
+}
+
+void std_aligned_free(void* ptr) {
+
+#if defined(POSIXALIGNEDALLOC)
+ free(ptr);
+#elif defined(_WIN32) && !defined(_M_ARM) && !defined(_M_ARM64)
+ _mm_free(ptr);
+#elif defined(_WIN32)
+ _aligned_free(ptr);
+#else
+ free(ptr);
+#endif
+}
+
+/// aligned_large_pages_alloc() will return suitably aligned memory, if possible using large pages.
+
+#if defined(_WIN32)
+
+static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize) {
+
+ #if !defined(_WIN64)
+ return nullptr;
+ #else
+
+ HANDLE hProcessToken { };
+ LUID luid { };
+ void* mem = nullptr;
+
+ const size_t largePageSize = GetLargePageMinimum();
+ if (!largePageSize)
+ return nullptr;
+
+ // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges
+
+ HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll"));
+
+ if (!hAdvapi32)
+ hAdvapi32 = LoadLibrary(TEXT("advapi32.dll"));
+
+ auto fun6 = fun6_t((void(*)())GetProcAddress(hAdvapi32, "OpenProcessToken"));
+ if (!fun6)
+ return nullptr;
+ auto fun7 = fun7_t((void(*)())GetProcAddress(hAdvapi32, "LookupPrivilegeValueA"));
+ if (!fun7)
+ return nullptr;
+ auto fun8 = fun8_t((void(*)())GetProcAddress(hAdvapi32, "AdjustTokenPrivileges"));
+ if (!fun8)
+ return nullptr;
+
+ // We need SeLockMemoryPrivilege, so try to enable it for the process
+ if (!fun6( // OpenProcessToken()
+ GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken))
+ return nullptr;
+
+ if (fun7( // LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &luid)
+ nullptr, "SeLockMemoryPrivilege", &luid))
+ {
+ TOKEN_PRIVILEGES tp { };
+ TOKEN_PRIVILEGES prevTp { };
+ DWORD prevTpLen = 0;
+
+ tp.PrivilegeCount = 1;
+ tp.Privileges[0].Luid = luid;
+ tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+
+ // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() succeeds,
+ // we still need to query GetLastError() to ensure that the privileges were actually obtained.
+ if (fun8( // AdjustTokenPrivileges()
+ hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, &prevTpLen) &&
+ GetLastError() == ERROR_SUCCESS)
+ {
+ // Round up size to full pages and allocate
+ allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1);
+ mem = VirtualAlloc(
+ nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
+
+ // Privilege no longer needed, restore previous state
+ fun8( // AdjustTokenPrivileges ()
+ hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr);
+ }
+ }
+
+ CloseHandle(hProcessToken);
+
+ return mem;
+
+ #endif
+}
+
+void* aligned_large_pages_alloc(size_t allocSize) {
+
+ // Try to allocate large pages
+ void* mem = aligned_large_pages_alloc_windows(allocSize);
+
+ // Fall back to regular, page aligned, allocation if necessary
+ if (!mem)
+ mem = VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+
+ return mem;
+}
+
+#else
+
+void* aligned_large_pages_alloc(size_t allocSize) {
+
+#if defined(__linux__)
+ constexpr size_t alignment = 2 * 1024 * 1024; // assumed 2MB page size
+#else
+ constexpr size_t alignment = 4096; // assumed small page size
+#endif
+
+ // round up to multiples of alignment
+ size_t size = ((allocSize + alignment - 1) / alignment) * alignment;
+ void *mem = std_aligned_alloc(alignment, size);
+#if defined(MADV_HUGEPAGE)
+ madvise(mem, size, MADV_HUGEPAGE);
+#endif
+ return mem;
+}
+
+#endif
+
+
+/// aligned_large_pages_free() will free the previously allocated ttmem
+
+#if defined(_WIN32)
+
+void aligned_large_pages_free(void* mem) {
+
+ if (mem && !VirtualFree(mem, 0, MEM_RELEASE))
+ {
+ DWORD err = GetLastError();
+ std::cerr << "Failed to free large page memory. Error code: 0x"
+ << std::hex << err
+ << std::dec << std::endl;
+ exit(EXIT_FAILURE);
+ }
+}
+
+#else
+
+void aligned_large_pages_free(void *mem) {
+ std_aligned_free(mem);